2024-11-19 16:14:08,225 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 16:14:08,236 main DEBUG Took 0.009776 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 16:14:08,237 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 16:14:08,237 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 16:14:08,238 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 16:14:08,239 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 16:14:08,246 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 16:14:08,259 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,261 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 16:14:08,262 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,263 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 16:14:08,264 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,264 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 16:14:08,265 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,266 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 16:14:08,267 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,267 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 16:14:08,268 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,269 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 16:14:08,269 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,270 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 16:14:08,271 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,271 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 16:14:08,272 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,272 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 16:14:08,273 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,273 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 16:14:08,274 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,274 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 16:14:08,275 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,275 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 16:14:08,276 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,276 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 16:14:08,278 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 16:14:08,280 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 16:14:08,283 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 16:14:08,284 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 16:14:08,286 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 16:14:08,286 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 16:14:08,296 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 16:14:08,298 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 16:14:08,300 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 16:14:08,300 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 16:14:08,301 main DEBUG createAppenders(={Console}) 2024-11-19 16:14:08,301 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-19 16:14:08,302 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 16:14:08,302 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-19 16:14:08,303 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 16:14:08,303 main DEBUG OutputStream closed 2024-11-19 16:14:08,303 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 16:14:08,303 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 16:14:08,304 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-19 16:14:08,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 16:14:08,379 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 16:14:08,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 16:14:08,381 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 16:14:08,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 16:14:08,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 16:14:08,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 16:14:08,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 16:14:08,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 16:14:08,384 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 16:14:08,384 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 16:14:08,385 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 16:14:08,385 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 16:14:08,385 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 16:14:08,386 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 16:14:08,386 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 16:14:08,386 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 16:14:08,387 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 16:14:08,389 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 16:14:08,389 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-19 16:14:08,389 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 16:14:08,390 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-19T16:14:08,593 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918 2024-11-19 16:14:08,596 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 16:14:08,597 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T16:14:08,604 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-19T16:14:08,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T16:14:08,640 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/cluster_f5154a92-f5a6-8acf-0581-31f31135a66d, deleteOnExit=true 2024-11-19T16:14:08,640 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T16:14:08,642 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/test.cache.data in system properties and HBase conf 2024-11-19T16:14:08,642 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T16:14:08,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/hadoop.log.dir in system properties and HBase conf 2024-11-19T16:14:08,644 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T16:14:08,644 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T16:14:08,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T16:14:08,732 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T16:14:08,826 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T16:14:08,831 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T16:14:08,831 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T16:14:08,832 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T16:14:08,832 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T16:14:08,833 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T16:14:08,833 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T16:14:08,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T16:14:08,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T16:14:08,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T16:14:08,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/nfs.dump.dir in system properties and HBase conf 2024-11-19T16:14:08,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/java.io.tmpdir in system properties and HBase conf 2024-11-19T16:14:08,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T16:14:08,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T16:14:08,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T16:14:09,911 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T16:14:09,989 INFO [Time-limited test {}] log.Log(170): Logging initialized @2449ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T16:14:10,055 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T16:14:10,113 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T16:14:10,131 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T16:14:10,132 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T16:14:10,133 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T16:14:10,142 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T16:14:10,145 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/hadoop.log.dir/,AVAILABLE} 2024-11-19T16:14:10,146 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T16:14:10,309 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/java.io.tmpdir/jetty-localhost-36717-hadoop-hdfs-3_4_1-tests_jar-_-any-10215390629411915178/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T16:14:10,315 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:36717} 2024-11-19T16:14:10,315 INFO [Time-limited test {}] server.Server(415): Started @2776ms 2024-11-19T16:14:10,912 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T16:14:10,919 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T16:14:10,921 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T16:14:10,921 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T16:14:10,921 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T16:14:10,922 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/hadoop.log.dir/,AVAILABLE} 2024-11-19T16:14:10,923 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T16:14:11,037 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/java.io.tmpdir/jetty-localhost-36321-hadoop-hdfs-3_4_1-tests_jar-_-any-9086369678985259335/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T16:14:11,038 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:36321} 2024-11-19T16:14:11,038 INFO [Time-limited test {}] server.Server(415): Started @3499ms 2024-11-19T16:14:11,084 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T16:14:12,124 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/cluster_f5154a92-f5a6-8acf-0581-31f31135a66d/data/data1/current/BP-1730201524-172.17.0.3-1732032849382/current, will proceed with Du for space computation calculation, 2024-11-19T16:14:12,124 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/cluster_f5154a92-f5a6-8acf-0581-31f31135a66d/data/data2/current/BP-1730201524-172.17.0.3-1732032849382/current, will proceed with Du for space computation calculation, 2024-11-19T16:14:12,217 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T16:14:12,292 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x215f834e454f4f71 with lease ID 0x90d2d7419c625a70: Processing first storage report for DS-79eeff2e-c28b-4213-ac21-88785b97fe77 from datanode DatanodeRegistration(127.0.0.1:41553, datanodeUuid=f686d3cb-c1c2-4c17-b87a-b78090b56ff8, infoPort=46497, infoSecurePort=0, ipcPort=42483, storageInfo=lv=-57;cid=testClusterID;nsid=569694886;c=1732032849382) 2024-11-19T16:14:12,294 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x215f834e454f4f71 with lease ID 0x90d2d7419c625a70: from storage DS-79eeff2e-c28b-4213-ac21-88785b97fe77 node DatanodeRegistration(127.0.0.1:41553, datanodeUuid=f686d3cb-c1c2-4c17-b87a-b78090b56ff8, infoPort=46497, infoSecurePort=0, ipcPort=42483, storageInfo=lv=-57;cid=testClusterID;nsid=569694886;c=1732032849382), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-19T16:14:12,294 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x215f834e454f4f71 with lease ID 0x90d2d7419c625a70: Processing first storage report for DS-5b2f02d3-78b2-4cb8-b334-3727b2248e4d from datanode DatanodeRegistration(127.0.0.1:41553, datanodeUuid=f686d3cb-c1c2-4c17-b87a-b78090b56ff8, infoPort=46497, infoSecurePort=0, ipcPort=42483, storageInfo=lv=-57;cid=testClusterID;nsid=569694886;c=1732032849382) 2024-11-19T16:14:12,295 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x215f834e454f4f71 with lease ID 0x90d2d7419c625a70: from storage DS-5b2f02d3-78b2-4cb8-b334-3727b2248e4d node DatanodeRegistration(127.0.0.1:41553, datanodeUuid=f686d3cb-c1c2-4c17-b87a-b78090b56ff8, infoPort=46497, infoSecurePort=0, ipcPort=42483, storageInfo=lv=-57;cid=testClusterID;nsid=569694886;c=1732032849382), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T16:14:12,302 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918 2024-11-19T16:14:12,377 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/cluster_f5154a92-f5a6-8acf-0581-31f31135a66d/zookeeper_0, clientPort=50373, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/cluster_f5154a92-f5a6-8acf-0581-31f31135a66d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/cluster_f5154a92-f5a6-8acf-0581-31f31135a66d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T16:14:12,390 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50373 2024-11-19T16:14:12,404 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T16:14:12,408 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T16:14:12,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741825_1001 (size=7) 2024-11-19T16:14:13,103 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70 with version=8 2024-11-19T16:14:13,104 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/hbase-staging 2024-11-19T16:14:13,190 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T16:14:13,390 INFO [Time-limited test {}] client.ConnectionUtils(128): master/32a8860bb9d6:0 server-side Connection retries=45 2024-11-19T16:14:13,399 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T16:14:13,399 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T16:14:13,404 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T16:14:13,404 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T16:14:13,404 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T16:14:13,548 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T16:14:13,602 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T16:14:13,614 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T16:14:13,618 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T16:14:13,648 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4693 (auto-detected) 2024-11-19T16:14:13,649 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-19T16:14:13,666 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37355 2024-11-19T16:14:13,687 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37355 connecting to ZooKeeper ensemble=127.0.0.1:50373 2024-11-19T16:14:13,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:373550x0, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T16:14:13,789 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37355-0x10153dfed330000 connected 2024-11-19T16:14:13,917 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T16:14:13,919 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T16:14:13,928 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T16:14:13,932 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70, hbase.cluster.distributed=false 2024-11-19T16:14:13,959 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T16:14:13,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37355 2024-11-19T16:14:13,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37355 2024-11-19T16:14:13,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37355 2024-11-19T16:14:13,965 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37355 2024-11-19T16:14:13,965 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37355 2024-11-19T16:14:14,083 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32a8860bb9d6:0 server-side Connection retries=45 2024-11-19T16:14:14,084 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T16:14:14,085 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T16:14:14,085 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T16:14:14,085 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T16:14:14,085 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T16:14:14,088 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T16:14:14,090 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T16:14:14,091 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43095 2024-11-19T16:14:14,093 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43095 connecting to ZooKeeper ensemble=127.0.0.1:50373 2024-11-19T16:14:14,094 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T16:14:14,098 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T16:14:14,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430950x0, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T16:14:14,163 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T16:14:14,163 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43095-0x10153dfed330001 connected 2024-11-19T16:14:14,168 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T16:14:14,175 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T16:14:14,177 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T16:14:14,182 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T16:14:14,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43095 2024-11-19T16:14:14,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43095 2024-11-19T16:14:14,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43095 2024-11-19T16:14:14,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43095 2024-11-19T16:14:14,185 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43095 2024-11-19T16:14:14,198 DEBUG [M:0;32a8860bb9d6:37355 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;32a8860bb9d6:37355 2024-11-19T16:14:14,199 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/32a8860bb9d6,37355,1732032853237 2024-11-19T16:14:14,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T16:14:14,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T16:14:14,259 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/32a8860bb9d6,37355,1732032853237 2024-11-19T16:14:14,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T16:14:14,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:14:14,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:14:14,365 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T16:14:14,368 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/32a8860bb9d6,37355,1732032853237 from backup master directory 2024-11-19T16:14:14,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T16:14:14,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/32a8860bb9d6,37355,1732032853237 2024-11-19T16:14:14,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T16:14:14,420 WARN [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T16:14:14,420 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=32a8860bb9d6,37355,1732032853237 2024-11-19T16:14:14,425 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T16:14:14,426 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T16:14:14,477 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/hbase.id] with ID: df036147-1459-4aa3-9a3b-991071397ee6 2024-11-19T16:14:14,477 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/.tmp/hbase.id 2024-11-19T16:14:14,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741826_1002 (size=42) 2024-11-19T16:14:14,889 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/.tmp/hbase.id]:[hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/hbase.id] 2024-11-19T16:14:14,940 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T16:14:14,945 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T16:14:14,963 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-19T16:14:15,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:14:15,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:14:15,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741827_1003 (size=196) 2024-11-19T16:14:15,487 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T16:14:15,489 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T16:14:15,503 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:15,507 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T16:14:15,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741828_1004 (size=1189) 2024-11-19T16:14:15,961 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store 2024-11-19T16:14:15,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741829_1005 (size=34) 2024-11-19T16:14:16,386 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T16:14:16,390 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:14:16,392 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T16:14:16,393 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T16:14:16,393 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T16:14:16,394 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T16:14:16,395 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T16:14:16,395 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T16:14:16,396 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732032856392Disabling compacts and flushes for region at 1732032856392Disabling writes for close at 1732032856395 (+3 ms)Writing region close event to WAL at 1732032856395Closed at 1732032856395 2024-11-19T16:14:16,399 WARN [master/32a8860bb9d6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/.initializing 2024-11-19T16:14:16,399 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/WALs/32a8860bb9d6,37355,1732032853237 2024-11-19T16:14:16,409 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T16:14:16,425 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32a8860bb9d6%2C37355%2C1732032853237, suffix=, logDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/WALs/32a8860bb9d6,37355,1732032853237, archiveDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/oldWALs, maxLogs=10 2024-11-19T16:14:16,451 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/WALs/32a8860bb9d6,37355,1732032853237/32a8860bb9d6%2C37355%2C1732032853237.1732032856429, exclude list is [], retry=0 2024-11-19T16:14:16,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41553,DS-79eeff2e-c28b-4213-ac21-88785b97fe77,DISK] 2024-11-19T16:14:16,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T16:14:16,509 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/WALs/32a8860bb9d6,37355,1732032853237/32a8860bb9d6%2C37355%2C1732032853237.1732032856429 2024-11-19T16:14:16,510 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46497:46497)] 2024-11-19T16:14:16,511 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T16:14:16,512 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:14:16,515 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T16:14:16,516 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T16:14:16,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T16:14:16,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T16:14:16,595 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:16,598 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T16:14:16,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T16:14:16,603 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T16:14:16,604 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:16,605 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:14:16,606 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T16:14:16,610 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T16:14:16,610 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:16,612 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:14:16,612 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T16:14:16,617 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T16:14:16,618 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:16,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:14:16,620 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T16:14:16,625 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T16:14:16,627 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T16:14:16,634 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T16:14:16,635 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T16:14:16,639 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T16:14:16,643 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T16:14:16,647 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T16:14:16,649 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73453607, jitterRate=0.09454403817653656}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T16:14:16,658 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732032856529Initializing all the Stores at 1732032856531 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732032856532 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032856533 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032856533Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032856533Cleaning up temporary data from old regions at 1732032856635 (+102 ms)Region opened successfully at 1732032856658 (+23 ms) 2024-11-19T16:14:16,660 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T16:14:16,700 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a8a344, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32a8860bb9d6/172.17.0.3:0 2024-11-19T16:14:16,738 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T16:14:16,749 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T16:14:16,750 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T16:14:16,752 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T16:14:16,754 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T16:14:16,758 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-19T16:14:16,758 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T16:14:16,790 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T16:14:16,800 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T16:14:16,809 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T16:14:16,813 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T16:14:16,815 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T16:14:16,828 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T16:14:16,831 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T16:14:16,836 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T16:14:16,851 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T16:14:16,853 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T16:14:16,872 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T16:14:16,890 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T16:14:16,904 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T16:14:16,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T16:14:16,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T16:14:16,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:14:16,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:14:16,930 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=32a8860bb9d6,37355,1732032853237, sessionid=0x10153dfed330000, setting cluster-up flag (Was=false) 2024-11-19T16:14:16,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:14:16,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:14:17,073 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T16:14:17,075 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32a8860bb9d6,37355,1732032853237 2024-11-19T16:14:17,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:14:17,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:14:17,135 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T16:14:17,139 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32a8860bb9d6,37355,1732032853237 2024-11-19T16:14:17,148 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T16:14:17,190 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(746): ClusterId : df036147-1459-4aa3-9a3b-991071397ee6 2024-11-19T16:14:17,193 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T16:14:17,210 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T16:14:17,210 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T16:14:17,215 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T16:14:17,221 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T16:14:17,222 DEBUG [RS:0;32a8860bb9d6:43095 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50a2ada5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32a8860bb9d6/172.17.0.3:0 2024-11-19T16:14:17,225 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T16:14:17,234 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T16:14:17,243 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;32a8860bb9d6:43095 2024-11-19T16:14:17,241 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 32a8860bb9d6,37355,1732032853237 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T16:14:17,246 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T16:14:17,247 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T16:14:17,247 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T16:14:17,249 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(2659): reportForDuty to master=32a8860bb9d6,37355,1732032853237 with port=43095, startcode=1732032854050 2024-11-19T16:14:17,250 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/32a8860bb9d6:0, corePoolSize=5, maxPoolSize=5 2024-11-19T16:14:17,250 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/32a8860bb9d6:0, corePoolSize=5, maxPoolSize=5 2024-11-19T16:14:17,250 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/32a8860bb9d6:0, corePoolSize=5, maxPoolSize=5 2024-11-19T16:14:17,251 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/32a8860bb9d6:0, corePoolSize=5, maxPoolSize=5 2024-11-19T16:14:17,251 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/32a8860bb9d6:0, corePoolSize=10, maxPoolSize=10 2024-11-19T16:14:17,251 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/32a8860bb9d6:0, corePoolSize=1, maxPoolSize=1 2024-11-19T16:14:17,251 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/32a8860bb9d6:0, corePoolSize=2, maxPoolSize=2 2024-11-19T16:14:17,251 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/32a8860bb9d6:0, corePoolSize=1, maxPoolSize=1 2024-11-19T16:14:17,258 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732032887258 2024-11-19T16:14:17,259 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T16:14:17,259 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T16:14:17,260 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T16:14:17,262 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T16:14:17,263 DEBUG [RS:0;32a8860bb9d6:43095 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T16:14:17,266 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T16:14:17,267 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T16:14:17,267 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:17,267 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T16:14:17,267 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T16:14:17,267 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T16:14:17,273 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,278 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T16:14:17,279 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T16:14:17,280 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T16:14:17,282 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T16:14:17,282 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T16:14:17,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741831_1007 (size=1321) 2024-11-19T16:14:17,292 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/32a8860bb9d6:0:becomeActiveMaster-HFileCleaner.large.0-1732032857283,5,FailOnTimeoutGroup] 2024-11-19T16:14:17,293 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/32a8860bb9d6:0:becomeActiveMaster-HFileCleaner.small.0-1732032857293,5,FailOnTimeoutGroup] 2024-11-19T16:14:17,294 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,294 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T16:14:17,295 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,296 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,311 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34609, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T16:14:17,317 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37355 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:17,319 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37355 {}] master.ServerManager(517): Registering regionserver=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:17,333 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70 2024-11-19T16:14:17,333 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36477 2024-11-19T16:14:17,333 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T16:14:17,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T16:14:17,347 DEBUG [RS:0;32a8860bb9d6:43095 {}] zookeeper.ZKUtil(111): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:17,347 WARN [RS:0;32a8860bb9d6:43095 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T16:14:17,347 INFO [RS:0;32a8860bb9d6:43095 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T16:14:17,347 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/WALs/32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:17,349 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32a8860bb9d6,43095,1732032854050] 2024-11-19T16:14:17,372 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T16:14:17,388 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T16:14:17,392 INFO [RS:0;32a8860bb9d6:43095 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T16:14:17,392 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,394 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T16:14:17,400 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T16:14:17,401 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,401 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32a8860bb9d6:0, corePoolSize=1, maxPoolSize=1 2024-11-19T16:14:17,401 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32a8860bb9d6:0, corePoolSize=1, maxPoolSize=1 2024-11-19T16:14:17,401 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32a8860bb9d6:0, corePoolSize=1, maxPoolSize=1 2024-11-19T16:14:17,401 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32a8860bb9d6:0, corePoolSize=1, maxPoolSize=1 2024-11-19T16:14:17,402 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32a8860bb9d6:0, corePoolSize=1, maxPoolSize=1 2024-11-19T16:14:17,402 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32a8860bb9d6:0, corePoolSize=2, maxPoolSize=2 2024-11-19T16:14:17,402 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32a8860bb9d6:0, corePoolSize=1, maxPoolSize=1 2024-11-19T16:14:17,402 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32a8860bb9d6:0, corePoolSize=1, maxPoolSize=1 2024-11-19T16:14:17,402 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32a8860bb9d6:0, corePoolSize=1, maxPoolSize=1 2024-11-19T16:14:17,402 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32a8860bb9d6:0, corePoolSize=1, maxPoolSize=1 2024-11-19T16:14:17,402 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32a8860bb9d6:0, corePoolSize=1, maxPoolSize=1 2024-11-19T16:14:17,403 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32a8860bb9d6:0, corePoolSize=1, maxPoolSize=1 2024-11-19T16:14:17,403 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32a8860bb9d6:0, corePoolSize=3, maxPoolSize=3 2024-11-19T16:14:17,403 DEBUG [RS:0;32a8860bb9d6:43095 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0, corePoolSize=3, maxPoolSize=3 2024-11-19T16:14:17,404 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,404 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,404 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,405 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,405 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,405 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a8860bb9d6,43095,1732032854050-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T16:14:17,425 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T16:14:17,426 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a8860bb9d6,43095,1732032854050-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,427 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,427 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.Replication(171): 32a8860bb9d6,43095,1732032854050 started 2024-11-19T16:14:17,445 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:17,446 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1482): Serving as 32a8860bb9d6,43095,1732032854050, RpcServer on 32a8860bb9d6/172.17.0.3:43095, sessionid=0x10153dfed330001 2024-11-19T16:14:17,446 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T16:14:17,447 DEBUG [RS:0;32a8860bb9d6:43095 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:17,447 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a8860bb9d6,43095,1732032854050' 2024-11-19T16:14:17,447 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T16:14:17,448 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T16:14:17,449 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T16:14:17,449 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T16:14:17,449 DEBUG [RS:0;32a8860bb9d6:43095 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:17,449 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a8860bb9d6,43095,1732032854050' 2024-11-19T16:14:17,449 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T16:14:17,450 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T16:14:17,451 DEBUG [RS:0;32a8860bb9d6:43095 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T16:14:17,451 INFO [RS:0;32a8860bb9d6:43095 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T16:14:17,451 INFO [RS:0;32a8860bb9d6:43095 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T16:14:17,556 INFO [RS:0;32a8860bb9d6:43095 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T16:14:17,559 INFO [RS:0;32a8860bb9d6:43095 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32a8860bb9d6%2C43095%2C1732032854050, suffix=, logDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/WALs/32a8860bb9d6,43095,1732032854050, archiveDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/oldWALs, maxLogs=32 2024-11-19T16:14:17,575 DEBUG [RS:0;32a8860bb9d6:43095 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/WALs/32a8860bb9d6,43095,1732032854050/32a8860bb9d6%2C43095%2C1732032854050.1732032857562, exclude list is [], retry=0 2024-11-19T16:14:17,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41553,DS-79eeff2e-c28b-4213-ac21-88785b97fe77,DISK] 2024-11-19T16:14:17,583 INFO [RS:0;32a8860bb9d6:43095 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/WALs/32a8860bb9d6,43095,1732032854050/32a8860bb9d6%2C43095%2C1732032854050.1732032857562 2024-11-19T16:14:17,584 DEBUG [RS:0;32a8860bb9d6:43095 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46497:46497)] 2024-11-19T16:14:17,687 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T16:14:17,688 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70 2024-11-19T16:14:17,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741833_1009 (size=32) 2024-11-19T16:14:18,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:14:18,107 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T16:14:18,111 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T16:14:18,111 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:18,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T16:14:18,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T16:14:18,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T16:14:18,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:18,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T16:14:18,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T16:14:18,120 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T16:14:18,120 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:18,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T16:14:18,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T16:14:18,123 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T16:14:18,123 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:18,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T16:14:18,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T16:14:18,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740 2024-11-19T16:14:18,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740 2024-11-19T16:14:18,129 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T16:14:18,129 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T16:14:18,130 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T16:14:18,132 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T16:14:18,136 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T16:14:18,137 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67280038, jitterRate=0.0025506913661956787}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T16:14:18,140 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732032858103Initializing all the Stores at 1732032858106 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732032858107 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732032858107Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032858107Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732032858107Cleaning up temporary data from old regions at 1732032858129 (+22 ms)Region opened successfully at 1732032858140 (+11 ms) 2024-11-19T16:14:18,140 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T16:14:18,140 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T16:14:18,140 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T16:14:18,140 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T16:14:18,140 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T16:14:18,142 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T16:14:18,142 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732032858140Disabling compacts and flushes for region at 1732032858140Disabling writes for close at 1732032858140Writing region close event to WAL at 1732032858141 (+1 ms)Closed at 1732032858142 (+1 ms) 2024-11-19T16:14:18,145 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T16:14:18,145 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T16:14:18,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T16:14:18,158 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T16:14:18,160 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T16:14:18,318 DEBUG [32a8860bb9d6:37355 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T16:14:18,330 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:18,338 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32a8860bb9d6,43095,1732032854050, state=OPENING 2024-11-19T16:14:18,388 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T16:14:18,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:14:18,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:14:18,402 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T16:14:18,402 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T16:14:18,405 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T16:14:18,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:14:18,587 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T16:14:18,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40773, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T16:14:18,602 INFO [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T16:14:18,602 INFO [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T16:14:18,603 INFO [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T16:14:18,606 INFO [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32a8860bb9d6%2C43095%2C1732032854050.meta, suffix=.meta, logDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/WALs/32a8860bb9d6,43095,1732032854050, archiveDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/oldWALs, maxLogs=32 2024-11-19T16:14:18,622 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/WALs/32a8860bb9d6,43095,1732032854050/32a8860bb9d6%2C43095%2C1732032854050.meta.1732032858608.meta, exclude list is [], retry=0 2024-11-19T16:14:18,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41553,DS-79eeff2e-c28b-4213-ac21-88785b97fe77,DISK] 2024-11-19T16:14:18,628 INFO [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/WALs/32a8860bb9d6,43095,1732032854050/32a8860bb9d6%2C43095%2C1732032854050.meta.1732032858608.meta 2024-11-19T16:14:18,629 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46497:46497)] 2024-11-19T16:14:18,629 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T16:14:18,631 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T16:14:18,633 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T16:14:18,637 INFO [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T16:14:18,642 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T16:14:18,642 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:14:18,642 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T16:14:18,643 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T16:14:18,645 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T16:14:18,647 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T16:14:18,647 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:18,648 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T16:14:18,648 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T16:14:18,649 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T16:14:18,649 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:18,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T16:14:18,651 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T16:14:18,652 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T16:14:18,652 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:18,653 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T16:14:18,653 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T16:14:18,654 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T16:14:18,655 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:18,655 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T16:14:18,656 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T16:14:18,657 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740 2024-11-19T16:14:18,660 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740 2024-11-19T16:14:18,662 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T16:14:18,662 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T16:14:18,663 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T16:14:18,666 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T16:14:18,667 INFO [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70219958, jitterRate=0.04635891318321228}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T16:14:18,667 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T16:14:18,669 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732032858643Writing region info on filesystem at 1732032858643Initializing all the Stores at 1732032858645 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732032858645Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732032858645Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032858645Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732032858645Cleaning up temporary data from old regions at 1732032858663 (+18 ms)Running coprocessor post-open hooks at 1732032858667 (+4 ms)Region opened successfully at 1732032858668 (+1 ms) 2024-11-19T16:14:18,675 INFO [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732032858580 2024-11-19T16:14:18,687 DEBUG [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T16:14:18,687 INFO [RS_OPEN_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T16:14:18,689 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:18,692 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32a8860bb9d6,43095,1732032854050, state=OPEN 2024-11-19T16:14:18,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T16:14:18,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T16:14:18,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T16:14:18,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T16:14:18,993 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:19,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T16:14:19,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=32a8860bb9d6,43095,1732032854050 in 585 msec 2024-11-19T16:14:19,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T16:14:19,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 854 msec 2024-11-19T16:14:19,012 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T16:14:19,012 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T16:14:19,033 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:19,034 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:19,054 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:19,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58511, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:19,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.9020 sec 2024-11-19T16:14:19,078 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732032859078, completionTime=-1 2024-11-19T16:14:19,081 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T16:14:19,081 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T16:14:19,123 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-19T16:14:19,123 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732032919123 2024-11-19T16:14:19,123 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732032979123 2024-11-19T16:14:19,123 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 41 msec 2024-11-19T16:14:19,125 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a8860bb9d6,37355,1732032853237-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:19,125 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a8860bb9d6,37355,1732032853237-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:19,125 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a8860bb9d6,37355,1732032853237-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:19,130 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-32a8860bb9d6:37355, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:19,130 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:19,131 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:19,136 DEBUG [master/32a8860bb9d6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T16:14:19,156 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.734sec 2024-11-19T16:14:19,157 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T16:14:19,157 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T16:14:19,158 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T16:14:19,159 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T16:14:19,159 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T16:14:19,159 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a8860bb9d6,37355,1732032853237-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T16:14:19,160 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a8860bb9d6,37355,1732032853237-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T16:14:19,168 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T16:14:19,169 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T16:14:19,169 INFO [master/32a8860bb9d6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a8860bb9d6,37355,1732032853237-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T16:14:19,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:19,221 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T16:14:19,221 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T16:14:19,223 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:19,226 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:19,232 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:19,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:19,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:19,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78d43c0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:19,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:19,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:19,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:19,240 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:19,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:19,243 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:19,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:19,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:19,275 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=32a8860bb9d6,37355,1732032853237 2024-11-19T16:14:19,285 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=4953 2024-11-19T16:14:19,294 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T16:14:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 32a8860bb9d6,37355,1732032853237 2024-11-19T16:14:19,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4ae4b090 2024-11-19T16:14:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T16:14:19,304 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T16:14:19,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T16:14:19,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T16:14:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T16:14:19,322 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T16:14:19,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-19T16:14:19,325 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:19,327 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T16:14:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T16:14:19,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741835_1011 (size=1061) 2024-11-19T16:14:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T16:14:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T16:14:19,748 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70 2024-11-19T16:14:19,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741836_1012 (size=53) 2024-11-19T16:14:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T16:14:20,161 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:14:20,161 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c20cca7a0cf0913bda0c7ed239b24e88, disabling compactions & flushes 2024-11-19T16:14:20,161 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:20,162 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:20,162 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. after waiting 0 ms 2024-11-19T16:14:20,162 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:20,162 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:20,162 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c20cca7a0cf0913bda0c7ed239b24e88: Waiting for close lock at 1732032860161Disabling compacts and flushes for region at 1732032860161Disabling writes for close at 1732032860162 (+1 ms)Writing region close event to WAL at 1732032860162Closed at 1732032860162 2024-11-19T16:14:20,168 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T16:14:20,175 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732032860168"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732032860168"}]},"ts":"1732032860168"} 2024-11-19T16:14:20,180 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T16:14:20,182 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T16:14:20,185 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032860182"}]},"ts":"1732032860182"} 2024-11-19T16:14:20,189 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T16:14:20,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c20cca7a0cf0913bda0c7ed239b24e88, ASSIGN}] 2024-11-19T16:14:20,194 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c20cca7a0cf0913bda0c7ed239b24e88, ASSIGN 2024-11-19T16:14:20,196 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c20cca7a0cf0913bda0c7ed239b24e88, ASSIGN; state=OFFLINE, location=32a8860bb9d6,43095,1732032854050; forceNewPlan=false, retain=false 2024-11-19T16:14:20,348 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c20cca7a0cf0913bda0c7ed239b24e88, regionState=OPENING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:20,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c20cca7a0cf0913bda0c7ed239b24e88, ASSIGN because future has completed 2024-11-19T16:14:20,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:14:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T16:14:20,522 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:20,522 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} 2024-11-19T16:14:20,523 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:20,523 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:14:20,523 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:20,523 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:20,526 INFO [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:20,529 INFO [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:14:20,530 INFO [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c20cca7a0cf0913bda0c7ed239b24e88 columnFamilyName A 2024-11-19T16:14:20,530 DEBUG [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:20,531 INFO [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] regionserver.HStore(327): Store=c20cca7a0cf0913bda0c7ed239b24e88/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:14:20,532 INFO [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:20,534 INFO [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:14:20,535 INFO [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c20cca7a0cf0913bda0c7ed239b24e88 columnFamilyName B 2024-11-19T16:14:20,535 DEBUG [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:20,536 INFO [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] regionserver.HStore(327): Store=c20cca7a0cf0913bda0c7ed239b24e88/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:14:20,537 INFO [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:20,539 INFO [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:14:20,540 INFO [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c20cca7a0cf0913bda0c7ed239b24e88 columnFamilyName C 2024-11-19T16:14:20,540 DEBUG [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:20,541 INFO [StoreOpener-c20cca7a0cf0913bda0c7ed239b24e88-1 {}] regionserver.HStore(327): Store=c20cca7a0cf0913bda0c7ed239b24e88/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:14:20,542 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:20,542 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:20,543 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:20,544 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:20,546 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:20,546 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:20,547 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T16:14:20,549 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:20,553 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T16:14:20,554 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c20cca7a0cf0913bda0c7ed239b24e88; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64457693, jitterRate=-0.039505526423454285}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T16:14:20,554 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:20,555 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c20cca7a0cf0913bda0c7ed239b24e88: Running coprocessor pre-open hook at 1732032860523Writing region info on filesystem at 1732032860523Initializing all the Stores at 1732032860525 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032860525Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032860526 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032860526Cleaning up temporary data from old regions at 1732032860546 (+20 ms)Running coprocessor post-open hooks at 1732032860554 (+8 ms)Region opened successfully at 1732032860554 2024-11-19T16:14:20,557 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., pid=6, masterSystemTime=1732032860510 2024-11-19T16:14:20,561 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:20,561 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:20,562 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c20cca7a0cf0913bda0c7ed239b24e88, regionState=OPEN, openSeqNum=2, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:20,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:14:20,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T16:14:20,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 in 213 msec 2024-11-19T16:14:20,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T16:14:20,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c20cca7a0cf0913bda0c7ed239b24e88, ASSIGN in 381 msec 2024-11-19T16:14:20,578 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T16:14:20,578 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032860578"}]},"ts":"1732032860578"} 2024-11-19T16:14:20,582 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T16:14:20,584 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T16:14:20,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2700 sec 2024-11-19T16:14:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T16:14:21,492 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:21,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:21,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:21,501 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2315f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:21,504 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35eddcd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,506 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:21,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:21,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:21,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:21,514 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T16:14:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 32a8860bb9d6,37355,1732032853237 2024-11-19T16:14:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@19bc5c32 2024-11-19T16:14:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T16:14:21,520 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T16:14:21,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:21,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:21,527 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7290076, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:21,530 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b17c100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,532 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,532 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:21,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:21,534 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f338c72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:21,538 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:21,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4184659e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:21,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:21,541 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:21,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:21,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b383f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:21,545 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:21,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:21,548 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ccb4eda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:21,552 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e934139, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:21,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:21,555 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8954be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:21,557 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260a5c9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:21,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:21,561 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510fbbc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:21,564 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62fa727e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:21,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:21,567 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35351cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:21,569 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516bf0d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:21,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:21,572 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9a1dcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:21,575 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ab586a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:21,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:21,579 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b6e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:21,581 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed66a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:21,587 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:21,587 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:21,587 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:21,587 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:21,588 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:21,588 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:21,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:21,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:21,591 DEBUG [Thread-160 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:21,592 DEBUG [Thread-160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:21,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:21,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:21,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:21,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:21,596 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:21,599 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:21,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:21,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:21,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:14:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:14:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:14:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:14:21,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:21,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:14:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:14:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:14:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:14:21,616 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:21,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:21,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:14:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-19T16:14:21,630 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:21,635 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:21,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T16:14:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:21,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:14:21,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:21,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:21,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:21,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:21,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:21,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T16:14:21,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/9ad8199da8b14b0d9c984f5b6ff54b34 is 50, key is test_row_0/A:col10/1732032861652/Put/seqid=0 2024-11-19T16:14:21,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741837_1013 (size=12001) 2024-11-19T16:14:21,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/9ad8199da8b14b0d9c984f5b6ff54b34 2024-11-19T16:14:21,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732032861753, max timestamp=1732032861790Num uniques 859; 2024-11-19T16:14:21,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T16:14:21,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:21,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:21,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:21,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:21,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:21,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:21,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032861753, max timestamp=1732032861790Num uniques 859; 2024-11-19T16:14:21,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:14:21,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732032861753, max timestamp=1732032861814Num uniques 859; 2024-11-19T16:14:21,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T16:14:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:21,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032921891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:21,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:21,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032921912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032921917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:21,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032921920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032921906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:21,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/7ce558c79a1040778685aec8271100b3 is 50, key is test_row_0/B:col10/1732032861652/Put/seqid=0 2024-11-19T16:14:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T16:14:21,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741838_1014 (size=12001) 2024-11-19T16:14:22,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:22,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:22,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032922058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032922059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032922059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032922059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032922060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:22,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T16:14:22,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032922277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032922278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032922278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032922279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032922279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:22,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:22,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/7ce558c79a1040778685aec8271100b3 2024-11-19T16:14:22,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/bbe3eae4be2c489cb28d7d2d27a04a70 is 50, key is test_row_0/C:col10/1732032861652/Put/seqid=0 2024-11-19T16:14:22,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741839_1015 (size=12001) 2024-11-19T16:14:22,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/bbe3eae4be2c489cb28d7d2d27a04a70 2024-11-19T16:14:22,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:22,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:22,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/9ad8199da8b14b0d9c984f5b6ff54b34 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9ad8199da8b14b0d9c984f5b6ff54b34 2024-11-19T16:14:22,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9ad8199da8b14b0d9c984f5b6ff54b34, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T16:14:22,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/7ce558c79a1040778685aec8271100b3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/7ce558c79a1040778685aec8271100b3 2024-11-19T16:14:22,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/7ce558c79a1040778685aec8271100b3, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T16:14:22,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/bbe3eae4be2c489cb28d7d2d27a04a70 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/bbe3eae4be2c489cb28d7d2d27a04a70 2024-11-19T16:14:22,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/bbe3eae4be2c489cb28d7d2d27a04a70, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T16:14:22,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=133.78 KB/136995 for c20cca7a0cf0913bda0c7ed239b24e88 in 910ms, sequenceid=12, compaction requested=false 2024-11-19T16:14:22,587 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T16:14:22,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:22,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=140.49 KB heapSize=399.77 KB 2024-11-19T16:14:22,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:22,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T16:14:22,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:22,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T16:14:22,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:22,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T16:14:22,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:22,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:22,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/a2188483714c402cb11707e6393e5759 is 50, key is test_row_0/A:col10/1732032861913/Put/seqid=0 2024-11-19T16:14:22,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741840_1016 (size=16681) 2024-11-19T16:14:22,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/a2188483714c402cb11707e6393e5759 2024-11-19T16:14:22,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032922643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032922650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032922681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/d45d947746de4a70b848ff4a6d9697fd is 50, key is test_row_0/B:col10/1732032861913/Put/seqid=0 2024-11-19T16:14:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032922689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032922694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741841_1017 (size=12001) 2024-11-19T16:14:22,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/d45d947746de4a70b848ff4a6d9697fd 2024-11-19T16:14:22,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/b51de2ddff694127bd9b3570b11f25bd is 50, key is test_row_0/C:col10/1732032861913/Put/seqid=0 2024-11-19T16:14:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T16:14:22,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741842_1018 (size=12001) 2024-11-19T16:14:22,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:22,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:22,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032922798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032922799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032922810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032922827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032922827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:22,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:22,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:22,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:22,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032923039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:23,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032923043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:23,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032923049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:23,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032923049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:23,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:23,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032923050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:23,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:23,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:23,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:23,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/b51de2ddff694127bd9b3570b11f25bd 2024-11-19T16:14:23,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/a2188483714c402cb11707e6393e5759 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/a2188483714c402cb11707e6393e5759 2024-11-19T16:14:23,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/a2188483714c402cb11707e6393e5759, entries=250, sequenceid=50, filesize=16.3 K 2024-11-19T16:14:23,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/d45d947746de4a70b848ff4a6d9697fd as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d45d947746de4a70b848ff4a6d9697fd 2024-11-19T16:14:23,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d45d947746de4a70b848ff4a6d9697fd, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T16:14:23,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/b51de2ddff694127bd9b3570b11f25bd as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/b51de2ddff694127bd9b3570b11f25bd 2024-11-19T16:14:23,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/b51de2ddff694127bd9b3570b11f25bd, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T16:14:23,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:23,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:23,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.20 KB/150735, heapSize ~419.88 KB/429960, currentSize=40.25 KB/41220 for c20cca7a0cf0913bda0c7ed239b24e88 in 666ms, sequenceid=50, compaction requested=false 2024-11-19T16:14:23,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:23,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:14:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:23,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:23,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:23,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:23,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:23,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:23,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/242903c45ccc4040bda619cd3ee07804 is 50, key is test_row_0/A:col10/1732032863350/Put/seqid=0 2024-11-19T16:14:23,367 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T16:14:23,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741843_1019 (size=14341) 2024-11-19T16:14:23,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/242903c45ccc4040bda619cd3ee07804 2024-11-19T16:14:23,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732032863387, max timestamp=1732032863419Num uniques 859; 2024-11-19T16:14:23,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T16:14:23,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032863387, max timestamp=1732032863419Num uniques 859; 2024-11-19T16:14:23,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:14:23,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/b8504b9010134e0aab296d8d3663f108 is 50, key is test_row_0/B:col10/1732032863350/Put/seqid=0 2024-11-19T16:14:23,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:23,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:23,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732032863387, max timestamp=1732032863419Num uniques 859; 2024-11-19T16:14:23,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T16:14:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741844_1020 (size=12001) 2024-11-19T16:14:23,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/b8504b9010134e0aab296d8d3663f108 2024-11-19T16:14:23,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032923476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:23,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032923477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:23,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032923480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:23,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032923484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:23,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:23,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032923487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/75734e56028b4c44aaa634f4477a8978 is 50, key is test_row_0/C:col10/1732032863350/Put/seqid=0 2024-11-19T16:14:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:23,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741845_1021 (size=12001) 2024-11-19T16:14:23,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/75734e56028b4c44aaa634f4477a8978 2024-11-19T16:14:23,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/242903c45ccc4040bda619cd3ee07804 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/242903c45ccc4040bda619cd3ee07804 2024-11-19T16:14:23,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/242903c45ccc4040bda619cd3ee07804, entries=200, sequenceid=60, filesize=14.0 K 2024-11-19T16:14:23,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/b8504b9010134e0aab296d8d3663f108 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b8504b9010134e0aab296d8d3663f108 2024-11-19T16:14:23,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b8504b9010134e0aab296d8d3663f108, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T16:14:23,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/75734e56028b4c44aaa634f4477a8978 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/75734e56028b4c44aaa634f4477a8978 2024-11-19T16:14:23,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/75734e56028b4c44aaa634f4477a8978, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T16:14:23,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:23,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:23,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,599 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T16:14:23,599 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T16:14:23,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.02 KB/139285 for c20cca7a0cf0913bda0c7ed239b24e88 in 247ms, sequenceid=60, compaction requested=true 2024-11-19T16:14:23,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:23,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:23,606 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:23,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:23,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:23,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:23,607 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:23,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:23,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:23,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=142.73 KB heapSize=404.62 KB 2024-11-19T16:14:23,610 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:23,612 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:23,612 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/B is initiating minor compaction (all files) 2024-11-19T16:14:23,612 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/A is initiating minor compaction (all files) 2024-11-19T16:14:23,613 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T16:14:23,613 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T16:14:23,613 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/B in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,613 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/7ce558c79a1040778685aec8271100b3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d45d947746de4a70b848ff4a6d9697fd, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b8504b9010134e0aab296d8d3663f108] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=35.2 K 2024-11-19T16:14:23,613 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T16:14:23,614 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T16:14:23,614 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/A in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,614 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9ad8199da8b14b0d9c984f5b6ff54b34, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/a2188483714c402cb11707e6393e5759, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/242903c45ccc4040bda619cd3ee07804] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=42.0 K 2024-11-19T16:14:23,615 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ce558c79a1040778685aec8271100b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732032861652 2024-11-19T16:14:23,615 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ad8199da8b14b0d9c984f5b6ff54b34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732032861652 2024-11-19T16:14:23,616 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2188483714c402cb11707e6393e5759, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732032861886 2024-11-19T16:14:23,616 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting d45d947746de4a70b848ff4a6d9697fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732032861913 2024-11-19T16:14:23,616 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T16:14:23,616 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T16:14:23,617 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 242903c45ccc4040bda619cd3ee07804, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732032862636 2024-11-19T16:14:23,617 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting b8504b9010134e0aab296d8d3663f108, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732032862680 2024-11-19T16:14:23,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732032863462, max timestamp=1732032863603Num uniques 1218; 2024-11-19T16:14:23,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732032863462, max timestamp=1732032863603Num uniques 1018; 2024-11-19T16:14:23,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732032863462, max timestamp=1732032863603Num uniques 1009; 2024-11-19T16:14:23,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=7688, suffix heap size=154640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=168 2024-11-19T16:14:23,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T16:14:23,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-19T16:14:23,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:23,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:23,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:23,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:23,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:23,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:23,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/b12403353a1f41708c80c57be394dee3 is 50, key is test_row_0/A:col10/1732032863599/Put/seqid=0 2024-11-19T16:14:23,675 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#A#compaction#10 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:23,676 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/18492a5bcd8d4d2389ea0b86d2117418 is 50, key is test_row_0/A:col10/1732032863350/Put/seqid=0 2024-11-19T16:14:23,679 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#B#compaction#11 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:23,679 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/ce03744cc750413aaeb564b7ea2b4138 is 50, key is test_row_0/B:col10/1732032863350/Put/seqid=0 2024-11-19T16:14:23,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741846_1022 (size=12001) 2024-11-19T16:14:23,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/b12403353a1f41708c80c57be394dee3 2024-11-19T16:14:23,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741847_1023 (size=12104) 2024-11-19T16:14:23,735 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/18492a5bcd8d4d2389ea0b86d2117418 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/18492a5bcd8d4d2389ea0b86d2117418 2024-11-19T16:14:23,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732032863686, max timestamp=1732032863733Num uniques 859; 2024-11-19T16:14:23,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732032863686, max timestamp=1732032863733Num uniques 859; 2024-11-19T16:14:23,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T16:14:23,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T16:14:23,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732032863686, max timestamp=1732032863733Num uniques 859; 2024-11-19T16:14:23,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T16:14:23,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/f1d36ac3ef024b729aab7df0efffdb05 is 50, key is test_row_0/B:col10/1732032863599/Put/seqid=0 2024-11-19T16:14:23,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:23,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:23,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741848_1024 (size=12104) 2024-11-19T16:14:23,783 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/ce03744cc750413aaeb564b7ea2b4138 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ce03744cc750413aaeb564b7ea2b4138 2024-11-19T16:14:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T16:14:23,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741849_1025 (size=12001) 2024-11-19T16:14:23,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/f1d36ac3ef024b729aab7df0efffdb05 2024-11-19T16:14:23,814 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/A of c20cca7a0cf0913bda0c7ed239b24e88 into 18492a5bcd8d4d2389ea0b86d2117418(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:23,814 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:23,818 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/A, priority=13, startTime=1732032863602; duration=0sec 2024-11-19T16:14:23,819 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:23,819 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:A 2024-11-19T16:14:23,819 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:23,828 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:23,829 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/C is initiating minor compaction (all files) 2024-11-19T16:14:23,829 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/C in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,829 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/bbe3eae4be2c489cb28d7d2d27a04a70, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/b51de2ddff694127bd9b3570b11f25bd, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/75734e56028b4c44aaa634f4477a8978] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=35.2 K 2024-11-19T16:14:23,831 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/B of c20cca7a0cf0913bda0c7ed239b24e88 into ce03744cc750413aaeb564b7ea2b4138(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:23,831 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:23,831 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/B, priority=13, startTime=1732032863606; duration=0sec 2024-11-19T16:14:23,831 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:23,831 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:B 2024-11-19T16:14:23,832 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbe3eae4be2c489cb28d7d2d27a04a70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732032861652 2024-11-19T16:14:23,833 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting b51de2ddff694127bd9b3570b11f25bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732032861913 2024-11-19T16:14:23,834 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75734e56028b4c44aaa634f4477a8978, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732032862680 2024-11-19T16:14:23,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/4b4e5bc1f2584a329e288ae160b9f9e6 is 50, key is test_row_0/C:col10/1732032863599/Put/seqid=0 2024-11-19T16:14:23,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032923849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:23,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732032863841, max timestamp=1732032863859Num uniques 1218; 2024-11-19T16:14:23,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=12268, suffix heap size=154640,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=268 2024-11-19T16:14:23,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732032863841, max timestamp=1732032863859Num uniques 1218; 2024-11-19T16:14:23,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=12268, suffix heap size=154640,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=268 2024-11-19T16:14:23,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732032863841, max timestamp=1732032863859Num uniques 1218; 2024-11-19T16:14:23,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=12268, suffix heap size=154640,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=268 2024-11-19T16:14:23,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741850_1026 (size=12001) 2024-11-19T16:14:23,878 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#C#compaction#14 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:23,879 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/e65e2ed2e89c42588b9e50d04d616a09 is 50, key is test_row_0/C:col10/1732032863350/Put/seqid=0 2024-11-19T16:14:23,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741851_1027 (size=12104) 2024-11-19T16:14:23,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:23,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:23,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:23,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:23,911 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/e65e2ed2e89c42588b9e50d04d616a09 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/e65e2ed2e89c42588b9e50d04d616a09 2024-11-19T16:14:23,925 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/C of c20cca7a0cf0913bda0c7ed239b24e88 into e65e2ed2e89c42588b9e50d04d616a09(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:23,925 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:23,926 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/C, priority=13, startTime=1732032863607; duration=0sec 2024-11-19T16:14:23,926 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:23,926 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:C 2024-11-19T16:14:23,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732032863912, max timestamp=1732032863935Num uniques 1127; 2024-11-19T16:14:23,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=14972, suffix heap size=145176,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=327 2024-11-19T16:14:23,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732032863912, max timestamp=1732032863935Num uniques 1127; 2024-11-19T16:14:23,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=14972, suffix heap size=145176,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=327 2024-11-19T16:14:23,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732032863912, max timestamp=1732032863935Num uniques 1127; 2024-11-19T16:14:23,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=14972, suffix heap size=145176,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=327 2024-11-19T16:14:23,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732032863975, max timestamp=1732032863988Num uniques 1186; 2024-11-19T16:14:23,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732032863975, max timestamp=1732032863988Num uniques 1186; 2024-11-19T16:14:23,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=13096, suffix heap size=151312,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=286 2024-11-19T16:14:23,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732032863975, max timestamp=1732032863988Num uniques 1186; 2024-11-19T16:14:23,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=13096, suffix heap size=151312,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=286 2024-11-19T16:14:23,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=13096, suffix heap size=151312,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=286 2024-11-19T16:14:24,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:24,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:24,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:24,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:24,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:24,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:24,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732032864070, max timestamp=1732032864088Num uniques 1145; 2024-11-19T16:14:24,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732032864070, max timestamp=1732032864088Num uniques 1145; 2024-11-19T16:14:24,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-19T16:14:24,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-19T16:14:24,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732032864070, max timestamp=1732032864088Num uniques 1145; 2024-11-19T16:14:24,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-19T16:14:24,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732032864135, max timestamp=1732032864153Num uniques 1104; 2024-11-19T16:14:24,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732032864135, max timestamp=1732032864153Num uniques 1104; 2024-11-19T16:14:24,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732032864135, max timestamp=1732032864153Num uniques 1104; 2024-11-19T16:14:24,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-11-19T16:14:24,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-11-19T16:14:24,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-11-19T16:14:24,167 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T16:14:24,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:24,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:24,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:24,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:24,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:24,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:24,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032924263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032924271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032924272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732032864256, max timestamp=1732032864271Num uniques 1163; 2024-11-19T16:14:24,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1732032864246, max timestamp=1732032864271Num uniques 1163; 2024-11-19T16:14:24,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=9748, suffix heap size=148920,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=213 2024-11-19T16:14:24,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=14328, suffix heap size=148920,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=313 2024-11-19T16:14:24,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732032864256, max timestamp=1732032864271Num uniques 1163; 2024-11-19T16:14:24,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=9748, suffix heap size=148920,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=213 2024-11-19T16:14:24,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.15 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/4b4e5bc1f2584a329e288ae160b9f9e6 2024-11-19T16:14:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/b12403353a1f41708c80c57be394dee3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/b12403353a1f41708c80c57be394dee3 2024-11-19T16:14:24,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/b12403353a1f41708c80c57be394dee3, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T16:14:24,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/f1d36ac3ef024b729aab7df0efffdb05 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/f1d36ac3ef024b729aab7df0efffdb05 2024-11-19T16:14:24,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/f1d36ac3ef024b729aab7df0efffdb05, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T16:14:24,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/4b4e5bc1f2584a329e288ae160b9f9e6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/4b4e5bc1f2584a329e288ae160b9f9e6 2024-11-19T16:14:24,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/4b4e5bc1f2584a329e288ae160b9f9e6, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T16:14:24,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732032864340, max timestamp=1732032864360Num uniques 1172; 2024-11-19T16:14:24,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=7872, suffix heap size=149856,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=172 2024-11-19T16:14:24,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732032864340, max timestamp=1732032864360Num uniques 1072; 2024-11-19T16:14:24,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732032864340, max timestamp=1732032864360Num uniques 1072; 2024-11-19T16:14:24,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-19T16:14:24,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-19T16:14:24,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.04 KB/26662, heapSize ~63.52 KB/65040, currentSize=26.84 KB/27480 for c20cca7a0cf0913bda0c7ed239b24e88 in 760ms, sequenceid=98, compaction requested=false 2024-11-19T16:14:24,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:24,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T16:14:24,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:24,379 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=46.96 KB heapSize=124.78 KB 2024-11-19T16:14:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:24,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:24,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:24,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/1cfe37efff3b4d0095dffc3437189ad9 is 50, key is test_row_0/A:col10/1732032864373/Put/seqid=0 2024-11-19T16:14:24,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741852_1028 (size=12151) 2024-11-19T16:14:24,415 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/1cfe37efff3b4d0095dffc3437189ad9 2024-11-19T16:14:24,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/81bca55a5afb48988baaff45ef04eb3d is 50, key is test_row_0/B:col10/1732032864373/Put/seqid=0 2024-11-19T16:14:24,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741853_1029 (size=12151) 2024-11-19T16:14:24,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1732032864459, max timestamp=1732032864474Num uniques 859; 2024-11-19T16:14:24,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732032864461, max timestamp=1732032864474Num uniques 859; 2024-11-19T16:14:24,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T16:14:24,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:14:24,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032864461, max timestamp=1732032864474Num uniques 859; 2024-11-19T16:14:24,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:14:24,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032924511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032924520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032924522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032924525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032924528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032924638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032924639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032924639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032924647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032924647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,840 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/81bca55a5afb48988baaff45ef04eb3d 2024-11-19T16:14:24,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/39cc7ac25e5146e6a610d017fa9f5f65 is 50, key is test_row_0/C:col10/1732032864373/Put/seqid=0 2024-11-19T16:14:24,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032924857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032924857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032924857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032924857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:24,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032924858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:24,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:24,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:24,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741854_1030 (size=12151) 2024-11-19T16:14:24,884 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/39cc7ac25e5146e6a610d017fa9f5f65 2024-11-19T16:14:24,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/1cfe37efff3b4d0095dffc3437189ad9 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/1cfe37efff3b4d0095dffc3437189ad9 2024-11-19T16:14:24,916 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/1cfe37efff3b4d0095dffc3437189ad9, entries=150, sequenceid=246, filesize=11.9 K 2024-11-19T16:14:24,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/81bca55a5afb48988baaff45ef04eb3d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/81bca55a5afb48988baaff45ef04eb3d 2024-11-19T16:14:24,935 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/81bca55a5afb48988baaff45ef04eb3d, entries=150, sequenceid=246, filesize=11.9 K 2024-11-19T16:14:24,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/39cc7ac25e5146e6a610d017fa9f5f65 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/39cc7ac25e5146e6a610d017fa9f5f65 2024-11-19T16:14:24,965 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/39cc7ac25e5146e6a610d017fa9f5f65, entries=150, sequenceid=246, filesize=11.9 K 2024-11-19T16:14:24,969 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~144.66 KB/148128, currentSize=138.65 KB/141980 for c20cca7a0cf0913bda0c7ed239b24e88 in 589ms, sequenceid=246, compaction requested=true 2024-11-19T16:14:24,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:24,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:24,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-19T16:14:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-19T16:14:24,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T16:14:24,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3450 sec 2024-11-19T16:14:24,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.3670 sec 2024-11-19T16:14:25,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:25,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=145.36 KB heapSize=410.38 KB 2024-11-19T16:14:25,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732032864519, max timestamp=1732032865172Num uniques 1218; 2024-11-19T16:14:25,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732032864519, max timestamp=1732032865172Num uniques 1068; 2024-11-19T16:14:25,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=7688, suffix heap size=154640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=168 2024-11-19T16:14:25,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-19T16:14:25,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:25,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:25,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:25,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:25,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:25,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T16:14:25,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/177b318a004544d4a7dfa43a4639448f is 50, key is test_row_0/A:col10/1732032864521/Put/seqid=0 2024-11-19T16:14:25,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741855_1031 (size=14741) 2024-11-19T16:14:25,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/177b318a004544d4a7dfa43a4639448f 2024-11-19T16:14:25,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032925227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032865209, max timestamp=1732032865239Num uniques 859; 2024-11-19T16:14:25,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032865209, max timestamp=1732032865239Num uniques 859; 2024-11-19T16:14:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032925233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:14:25,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:14:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032865209, max timestamp=1732032865239Num uniques 859; 2024-11-19T16:14:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:14:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/202d06a034774a40a1ab64b8cb0b0a13 is 50, key is test_row_0/B:col10/1732032864521/Put/seqid=0 2024-11-19T16:14:25,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741856_1032 (size=12301) 2024-11-19T16:14:25,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/202d06a034774a40a1ab64b8cb0b0a13 2024-11-19T16:14:25,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/90142d7e5d5542ba8cc2e8a65e2fe8b3 is 50, key is test_row_0/C:col10/1732032864521/Put/seqid=0 2024-11-19T16:14:25,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032925292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032925293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032925299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741857_1033 (size=12301) 2024-11-19T16:14:25,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/90142d7e5d5542ba8cc2e8a65e2fe8b3 2024-11-19T16:14:25,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/177b318a004544d4a7dfa43a4639448f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/177b318a004544d4a7dfa43a4639448f 2024-11-19T16:14:25,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032925347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032925357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/177b318a004544d4a7dfa43a4639448f, entries=200, sequenceid=284, filesize=14.4 K 2024-11-19T16:14:25,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/202d06a034774a40a1ab64b8cb0b0a13 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/202d06a034774a40a1ab64b8cb0b0a13 2024-11-19T16:14:25,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/202d06a034774a40a1ab64b8cb0b0a13, entries=150, sequenceid=284, filesize=12.0 K 2024-11-19T16:14:25,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/90142d7e5d5542ba8cc2e8a65e2fe8b3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/90142d7e5d5542ba8cc2e8a65e2fe8b3 2024-11-19T16:14:25,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/90142d7e5d5542ba8cc2e8a65e2fe8b3, entries=150, sequenceid=284, filesize=12.0 K 2024-11-19T16:14:25,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~178.75 KB/183040, currentSize=120.76 KB/123660 for c20cca7a0cf0913bda0c7ed239b24e88 in 237ms, sequenceid=284, compaction requested=true 2024-11-19T16:14:25,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:25,412 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:25,415 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:25,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:25,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:25,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:25,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:25,416 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50997 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:25,416 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/A is initiating minor compaction (all files) 2024-11-19T16:14:25,416 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/A in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:25,416 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/18492a5bcd8d4d2389ea0b86d2117418, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/b12403353a1f41708c80c57be394dee3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/1cfe37efff3b4d0095dffc3437189ad9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/177b318a004544d4a7dfa43a4639448f] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=49.8 K 2024-11-19T16:14:25,418 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18492a5bcd8d4d2389ea0b86d2117418, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732032862680 2024-11-19T16:14:25,418 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48557 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:25,418 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/B is initiating minor compaction (all files) 2024-11-19T16:14:25,418 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/B in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:25,419 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ce03744cc750413aaeb564b7ea2b4138, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/f1d36ac3ef024b729aab7df0efffdb05, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/81bca55a5afb48988baaff45ef04eb3d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/202d06a034774a40a1ab64b8cb0b0a13] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=47.4 K 2024-11-19T16:14:25,420 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting ce03744cc750413aaeb564b7ea2b4138, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732032862680 2024-11-19T16:14:25,420 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting b12403353a1f41708c80c57be394dee3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732032863483 2024-11-19T16:14:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:25,421 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cfe37efff3b4d0095dffc3437189ad9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732032864373 2024-11-19T16:14:25,421 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting f1d36ac3ef024b729aab7df0efffdb05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732032863483 2024-11-19T16:14:25,422 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 177b318a004544d4a7dfa43a4639448f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732032864519 2024-11-19T16:14:25,422 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 81bca55a5afb48988baaff45ef04eb3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732032864373 2024-11-19T16:14:25,423 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 202d06a034774a40a1ab64b8cb0b0a13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732032864521 2024-11-19T16:14:25,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-19T16:14:25,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:25,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:25,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:25,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:25,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:25,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:25,454 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#A#compaction#21 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:25,455 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/928355d0c14d4f379ad6872fea778d70 is 50, key is test_row_0/A:col10/1732032864521/Put/seqid=0 2024-11-19T16:14:25,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/d0394ae621d84b428f2cee27c50add7a is 50, key is test_row_0/A:col10/1732032865418/Put/seqid=0 2024-11-19T16:14:25,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032925456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032925456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,477 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#B#compaction#23 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:25,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,478 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/b78dbec3945e4b149ed814c1c127aca7 is 50, key is test_row_0/B:col10/1732032864521/Put/seqid=0 2024-11-19T16:14:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032925459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741859_1035 (size=14741) 2024-11-19T16:14:25,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741858_1034 (size=12541) 2024-11-19T16:14:25,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741860_1036 (size=12541) 2024-11-19T16:14:25,538 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/b78dbec3945e4b149ed814c1c127aca7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b78dbec3945e4b149ed814c1c127aca7 2024-11-19T16:14:25,551 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/B of c20cca7a0cf0913bda0c7ed239b24e88 into b78dbec3945e4b149ed814c1c127aca7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:25,551 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:25,551 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/B, priority=12, startTime=1732032865414; duration=0sec 2024-11-19T16:14:25,551 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:25,551 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:B 2024-11-19T16:14:25,551 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:25,556 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48557 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:25,556 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/C is initiating minor compaction (all files) 2024-11-19T16:14:25,556 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/C in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:25,556 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/e65e2ed2e89c42588b9e50d04d616a09, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/4b4e5bc1f2584a329e288ae160b9f9e6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/39cc7ac25e5146e6a610d017fa9f5f65, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/90142d7e5d5542ba8cc2e8a65e2fe8b3] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=47.4 K 2024-11-19T16:14:25,557 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting e65e2ed2e89c42588b9e50d04d616a09, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732032862680 2024-11-19T16:14:25,560 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b4e5bc1f2584a329e288ae160b9f9e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732032863483 2024-11-19T16:14:25,562 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 39cc7ac25e5146e6a610d017fa9f5f65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732032864373 2024-11-19T16:14:25,563 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 90142d7e5d5542ba8cc2e8a65e2fe8b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732032864521 2024-11-19T16:14:25,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032925567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032925578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032925578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032925567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032925587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,594 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#C#compaction#24 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:25,595 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/632f2e9ab9d24c71b5f8118129aadd53 is 50, key is test_row_0/C:col10/1732032864521/Put/seqid=0 2024-11-19T16:14:25,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741861_1037 (size=12541) 2024-11-19T16:14:25,686 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/632f2e9ab9d24c71b5f8118129aadd53 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/632f2e9ab9d24c71b5f8118129aadd53 2024-11-19T16:14:25,703 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/C of c20cca7a0cf0913bda0c7ed239b24e88 into 632f2e9ab9d24c71b5f8118129aadd53(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:25,703 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:25,703 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/C, priority=12, startTime=1732032865415; duration=0sec 2024-11-19T16:14:25,703 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:25,703 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:C 2024-11-19T16:14:25,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032925787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032925787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T16:14:25,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,798 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032925797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-19T16:14:25,802 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T16:14:25,803 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:25,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:25,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/d0394ae621d84b428f2cee27c50add7a 2024-11-19T16:14:25,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032925887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:25,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032925887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:25,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T16:14:25,910 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/928355d0c14d4f379ad6872fea778d70 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/928355d0c14d4f379ad6872fea778d70 2024-11-19T16:14:25,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/327e091321a342aca9375f82b037196b is 50, key is test_row_0/B:col10/1732032865418/Put/seqid=0 2024-11-19T16:14:25,928 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/A of c20cca7a0cf0913bda0c7ed239b24e88 into 928355d0c14d4f379ad6872fea778d70(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:25,928 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:25,929 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/A, priority=12, startTime=1732032865412; duration=0sec 2024-11-19T16:14:25,929 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:25,929 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:A 2024-11-19T16:14:25,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741862_1038 (size=12301) 2024-11-19T16:14:25,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/327e091321a342aca9375f82b037196b 2024-11-19T16:14:25,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T16:14:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:25,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:25,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/9bc15e56f6db4d5796b77a2b7592b715 is 50, key is test_row_0/C:col10/1732032865418/Put/seqid=0 2024-11-19T16:14:25,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741863_1039 (size=12301) 2024-11-19T16:14:26,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/9bc15e56f6db4d5796b77a2b7592b715 2024-11-19T16:14:26,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/d0394ae621d84b428f2cee27c50add7a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/d0394ae621d84b428f2cee27c50add7a 2024-11-19T16:14:26,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/d0394ae621d84b428f2cee27c50add7a, entries=200, sequenceid=320, filesize=14.4 K 2024-11-19T16:14:26,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/327e091321a342aca9375f82b037196b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/327e091321a342aca9375f82b037196b 2024-11-19T16:14:26,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/327e091321a342aca9375f82b037196b, entries=150, sequenceid=320, filesize=12.0 K 2024-11-19T16:14:26,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/9bc15e56f6db4d5796b77a2b7592b715 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/9bc15e56f6db4d5796b77a2b7592b715 2024-11-19T16:14:26,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/9bc15e56f6db4d5796b77a2b7592b715, entries=150, sequenceid=320, filesize=12.0 K 2024-11-19T16:14:26,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for c20cca7a0cf0913bda0c7ed239b24e88 in 648ms, sequenceid=320, compaction requested=false 2024-11-19T16:14:26,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:26,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:14:26,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:26,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:26,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:26,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:26,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:26,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:26,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/e84a094b224545e08c78980324607366 is 50, key is test_row_0/A:col10/1732032865456/Put/seqid=0 2024-11-19T16:14:26,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T16:14:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:26,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T16:14:26,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741864_1040 (size=14737) 2024-11-19T16:14:26,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/e84a094b224545e08c78980324607366 2024-11-19T16:14:26,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/d0abc96a0c6942349dcf78e4bd52dce1 is 50, key is test_row_0/B:col10/1732032865456/Put/seqid=0 2024-11-19T16:14:26,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032866143, max timestamp=1732032866156Num uniques 859; 2024-11-19T16:14:26,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032866143, max timestamp=1732032866156Num uniques 859; 2024-11-19T16:14:26,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:14:26,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:14:26,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032866143, max timestamp=1732032866156Num uniques 859; 2024-11-19T16:14:26,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:14:26,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741865_1041 (size=9857) 2024-11-19T16:14:26,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/d0abc96a0c6942349dcf78e4bd52dce1 2024-11-19T16:14:26,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/1064ea77dbc94cfc980fcddb8fdcab79 is 50, key is test_row_0/C:col10/1732032865456/Put/seqid=0 2024-11-19T16:14:26,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741866_1042 (size=9857) 2024-11-19T16:14:26,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/1064ea77dbc94cfc980fcddb8fdcab79 2024-11-19T16:14:26,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032926204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032926211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032926214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/e84a094b224545e08c78980324607366 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e84a094b224545e08c78980324607366 2024-11-19T16:14:26,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e84a094b224545e08c78980324607366, entries=200, sequenceid=334, filesize=14.4 K 2024-11-19T16:14:26,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/d0abc96a0c6942349dcf78e4bd52dce1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d0abc96a0c6942349dcf78e4bd52dce1 2024-11-19T16:14:26,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d0abc96a0c6942349dcf78e4bd52dce1, entries=100, sequenceid=334, filesize=9.6 K 2024-11-19T16:14:26,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/1064ea77dbc94cfc980fcddb8fdcab79 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1064ea77dbc94cfc980fcddb8fdcab79 2024-11-19T16:14:26,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1064ea77dbc94cfc980fcddb8fdcab79, entries=100, sequenceid=334, filesize=9.6 K 2024-11-19T16:14:26,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c20cca7a0cf0913bda0c7ed239b24e88 in 170ms, sequenceid=334, compaction requested=true 2024-11-19T16:14:26,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T16:14:26,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:26,270 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:26,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:26,270 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T16:14:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:26,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:26,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:26,273 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42019 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:26,273 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/A is initiating minor compaction (all files) 2024-11-19T16:14:26,273 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/A in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:26,273 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/928355d0c14d4f379ad6872fea778d70, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/d0394ae621d84b428f2cee27c50add7a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e84a094b224545e08c78980324607366] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=41.0 K 2024-11-19T16:14:26,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:26,273 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:26,274 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 928355d0c14d4f379ad6872fea778d70, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732032864521 2024-11-19T16:14:26,275 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0394ae621d84b428f2cee27c50add7a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732032865245 2024-11-19T16:14:26,276 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting e84a094b224545e08c78980324607366, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732032865453 2024-11-19T16:14:26,277 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:26,278 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/B is initiating minor compaction (all files) 2024-11-19T16:14:26,278 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/B in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:26,278 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b78dbec3945e4b149ed814c1c127aca7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/327e091321a342aca9375f82b037196b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d0abc96a0c6942349dcf78e4bd52dce1] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=33.9 K 2024-11-19T16:14:26,280 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting b78dbec3945e4b149ed814c1c127aca7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732032864521 2024-11-19T16:14:26,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:26,281 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 327e091321a342aca9375f82b037196b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732032865245 2024-11-19T16:14:26,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:26,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:26,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:26,286 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting d0abc96a0c6942349dcf78e4bd52dce1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732032865456 2024-11-19T16:14:26,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/41861233edc4497d92dd3ed0bbcb8cd5 is 50, key is test_row_0/A:col10/1732032866187/Put/seqid=0 2024-11-19T16:14:26,311 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:26,312 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/eebe4b55779542df889e4073435d5b83 is 50, key is test_row_0/A:col10/1732032865456/Put/seqid=0 2024-11-19T16:14:26,320 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#B#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:26,321 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/492b94e1e4e346c6baaf2ba796bc5f51 is 50, key is test_row_0/B:col10/1732032865456/Put/seqid=0 2024-11-19T16:14:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:26,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:26,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741868_1044 (size=12643) 2024-11-19T16:14:26,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741867_1043 (size=12301) 2024-11-19T16:14:26,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741869_1045 (size=12643) 2024-11-19T16:14:26,370 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/492b94e1e4e346c6baaf2ba796bc5f51 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/492b94e1e4e346c6baaf2ba796bc5f51 2024-11-19T16:14:26,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032926367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032926367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032926369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,383 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/B of c20cca7a0cf0913bda0c7ed239b24e88 into 492b94e1e4e346c6baaf2ba796bc5f51(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:26,383 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:26,383 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/B, priority=13, startTime=1732032866273; duration=0sec 2024-11-19T16:14:26,383 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:26,383 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:B 2024-11-19T16:14:26,383 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:26,385 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:26,385 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/C is initiating minor compaction (all files) 2024-11-19T16:14:26,386 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/C in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:26,386 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/632f2e9ab9d24c71b5f8118129aadd53, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/9bc15e56f6db4d5796b77a2b7592b715, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1064ea77dbc94cfc980fcddb8fdcab79] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=33.9 K 2024-11-19T16:14:26,388 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 632f2e9ab9d24c71b5f8118129aadd53, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732032864521 2024-11-19T16:14:26,389 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bc15e56f6db4d5796b77a2b7592b715, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732032865245 2024-11-19T16:14:26,390 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 1064ea77dbc94cfc980fcddb8fdcab79, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732032865456 2024-11-19T16:14:26,406 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:26,407 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/015d60bcb91541f593222e65e317a920 is 50, key is test_row_0/C:col10/1732032865456/Put/seqid=0 2024-11-19T16:14:26,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032926408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741870_1046 (size=12643) 2024-11-19T16:14:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032926408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T16:14:26,431 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/015d60bcb91541f593222e65e317a920 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/015d60bcb91541f593222e65e317a920 2024-11-19T16:14:26,442 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/C of c20cca7a0cf0913bda0c7ed239b24e88 into 015d60bcb91541f593222e65e317a920(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:26,442 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:26,442 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/C, priority=13, startTime=1732032866281; duration=0sec 2024-11-19T16:14:26,443 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:26,443 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:C 2024-11-19T16:14:26,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032926487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032926488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032926488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032926697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032926697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032926697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:26,746 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/41861233edc4497d92dd3ed0bbcb8cd5 2024-11-19T16:14:26,754 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/eebe4b55779542df889e4073435d5b83 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/eebe4b55779542df889e4073435d5b83 2024-11-19T16:14:26,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/e068eb7f61f64930b1c4b1e1131fca74 is 50, key is test_row_0/B:col10/1732032866187/Put/seqid=0 2024-11-19T16:14:26,775 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/A of c20cca7a0cf0913bda0c7ed239b24e88 into eebe4b55779542df889e4073435d5b83(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:26,775 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:26,775 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/A, priority=13, startTime=1732032866270; duration=0sec 2024-11-19T16:14:26,775 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:26,775 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:A 2024-11-19T16:14:26,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741871_1047 (size=12301) 2024-11-19T16:14:26,795 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/e068eb7f61f64930b1c4b1e1131fca74 2024-11-19T16:14:26,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/06c6134acb814a9faa015ad52a1d6338 is 50, key is test_row_0/C:col10/1732032866187/Put/seqid=0 2024-11-19T16:14:26,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741872_1048 (size=12301) 2024-11-19T16:14:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T16:14:27,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032927007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:27,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032927007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032927007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:27,234 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/06c6134acb814a9faa015ad52a1d6338 2024-11-19T16:14:27,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/41861233edc4497d92dd3ed0bbcb8cd5 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/41861233edc4497d92dd3ed0bbcb8cd5 2024-11-19T16:14:27,255 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/41861233edc4497d92dd3ed0bbcb8cd5, entries=150, sequenceid=369, filesize=12.0 K 2024-11-19T16:14:27,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/e068eb7f61f64930b1c4b1e1131fca74 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/e068eb7f61f64930b1c4b1e1131fca74 2024-11-19T16:14:27,271 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/e068eb7f61f64930b1c4b1e1131fca74, entries=150, sequenceid=369, filesize=12.0 K 2024-11-19T16:14:27,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/06c6134acb814a9faa015ad52a1d6338 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/06c6134acb814a9faa015ad52a1d6338 2024-11-19T16:14:27,284 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/06c6134acb814a9faa015ad52a1d6338, entries=150, sequenceid=369, filesize=12.0 K 2024-11-19T16:14:27,286 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for c20cca7a0cf0913bda0c7ed239b24e88 in 1016ms, sequenceid=369, compaction requested=false 2024-11-19T16:14:27,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:27,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:27,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-19T16:14:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-19T16:14:27,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-19T16:14:27,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4870 sec 2024-11-19T16:14:27,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.4980 sec 2024-11-19T16:14:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:27,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T16:14:27,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:27,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/e798660a544c4354bbeb4c3c212b7a61 is 50, key is test_row_0/A:col10/1732032867433/Put/seqid=0 2024-11-19T16:14:27,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741873_1049 (size=12301) 2024-11-19T16:14:27,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/e798660a544c4354bbeb4c3c212b7a61 2024-11-19T16:14:27,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/8b4e72517aff47fe89ab9d2cd5fd4491 is 50, key is test_row_0/B:col10/1732032867433/Put/seqid=0 2024-11-19T16:14:27,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741874_1050 (size=12301) 2024-11-19T16:14:27,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/8b4e72517aff47fe89ab9d2cd5fd4491 2024-11-19T16:14:27,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/9e5fb03da8d74f23a277b958d6782d4f is 50, key is test_row_0/C:col10/1732032867433/Put/seqid=0 2024-11-19T16:14:27,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741875_1051 (size=12301) 2024-11-19T16:14:27,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/9e5fb03da8d74f23a277b958d6782d4f 2024-11-19T16:14:27,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032927529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:27,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032927529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:27,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032867502, max timestamp=1732032867539Num uniques 859; 2024-11-19T16:14:27,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/e798660a544c4354bbeb4c3c212b7a61 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e798660a544c4354bbeb4c3c212b7a61 2024-11-19T16:14:27,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:14:27,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032867502, max timestamp=1732032867539Num uniques 859; 2024-11-19T16:14:27,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:14:27,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032867502, max timestamp=1732032867539Num uniques 859; 2024-11-19T16:14:27,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:14:27,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e798660a544c4354bbeb4c3c212b7a61, entries=150, sequenceid=385, filesize=12.0 K 2024-11-19T16:14:27,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/8b4e72517aff47fe89ab9d2cd5fd4491 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8b4e72517aff47fe89ab9d2cd5fd4491 2024-11-19T16:14:27,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8b4e72517aff47fe89ab9d2cd5fd4491, entries=150, sequenceid=385, filesize=12.0 K 2024-11-19T16:14:27,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/9e5fb03da8d74f23a277b958d6782d4f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/9e5fb03da8d74f23a277b958d6782d4f 2024-11-19T16:14:27,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/9e5fb03da8d74f23a277b958d6782d4f, entries=150, sequenceid=385, filesize=12.0 K 2024-11-19T16:14:27,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=73.80 KB/75570 for c20cca7a0cf0913bda0c7ed239b24e88 in 164ms, sequenceid=385, compaction requested=true 2024-11-19T16:14:27,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:27,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:27,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:27,601 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:27,601 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:27,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:27,603 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:27,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:27,603 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/B is initiating minor compaction (all files) 2024-11-19T16:14:27,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:27,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:27,603 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/B in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:27,603 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/492b94e1e4e346c6baaf2ba796bc5f51, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/e068eb7f61f64930b1c4b1e1131fca74, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8b4e72517aff47fe89ab9d2cd5fd4491] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=36.4 K 2024-11-19T16:14:27,604 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:27,604 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/A is initiating minor compaction (all files) 2024-11-19T16:14:27,604 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/A in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:27,604 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492b94e1e4e346c6baaf2ba796bc5f51, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732032865245 2024-11-19T16:14:27,604 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/eebe4b55779542df889e4073435d5b83, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/41861233edc4497d92dd3ed0bbcb8cd5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e798660a544c4354bbeb4c3c212b7a61] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=36.4 K 2024-11-19T16:14:27,605 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting eebe4b55779542df889e4073435d5b83, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732032865245 2024-11-19T16:14:27,605 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting e068eb7f61f64930b1c4b1e1131fca74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732032866187 2024-11-19T16:14:27,605 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 41861233edc4497d92dd3ed0bbcb8cd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732032866187 2024-11-19T16:14:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:27,606 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b4e72517aff47fe89ab9d2cd5fd4491, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732032866358 2024-11-19T16:14:27,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=80.51 KB heapSize=220.92 KB 2024-11-19T16:14:27,606 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting e798660a544c4354bbeb4c3c212b7a61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732032866358 2024-11-19T16:14:27,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:27,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:27,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:27,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:27,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:27,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:27,631 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#A#compaction#39 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:27,632 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/db29f33fa1cf4443878dca53c68afa12 is 50, key is test_row_0/A:col10/1732032867433/Put/seqid=0 2024-11-19T16:14:27,634 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:27,635 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/d3a2c791aed448538d5cd98093a73607 is 50, key is test_row_0/B:col10/1732032867433/Put/seqid=0 2024-11-19T16:14:27,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/9e4d51e1b8cb454a97e49236296d3265 is 50, key is test_row_0/A:col10/1732032867581/Put/seqid=0 2024-11-19T16:14:27,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741878_1054 (size=12745) 2024-11-19T16:14:27,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032927660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032927679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:27,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032927684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:27,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032927684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:27,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741877_1053 (size=12745) 2024-11-19T16:14:27,696 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/d3a2c791aed448538d5cd98093a73607 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d3a2c791aed448538d5cd98093a73607 2024-11-19T16:14:27,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741876_1052 (size=14741) 2024-11-19T16:14:27,709 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/B of c20cca7a0cf0913bda0c7ed239b24e88 into d3a2c791aed448538d5cd98093a73607(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:27,709 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:27,709 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/B, priority=13, startTime=1732032867601; duration=0sec 2024-11-19T16:14:27,709 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:27,709 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:B 2024-11-19T16:14:27,709 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:27,716 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:27,716 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/C is initiating minor compaction (all files) 2024-11-19T16:14:27,716 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/C in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:27,716 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/015d60bcb91541f593222e65e317a920, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/06c6134acb814a9faa015ad52a1d6338, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/9e5fb03da8d74f23a277b958d6782d4f] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=36.4 K 2024-11-19T16:14:27,717 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 015d60bcb91541f593222e65e317a920, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732032865245 2024-11-19T16:14:27,718 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06c6134acb814a9faa015ad52a1d6338, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732032866187 2024-11-19T16:14:27,719 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e5fb03da8d74f23a277b958d6782d4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732032866358 2024-11-19T16:14:27,753 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:27,753 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/c12154bb191b48dd96edf85efa816c9b is 50, key is test_row_0/C:col10/1732032867433/Put/seqid=0 2024-11-19T16:14:27,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741879_1055 (size=12745) 2024-11-19T16:14:27,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032927787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:27,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032927797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:27,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:27,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032927797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032927797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T16:14:27,947 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:27,948 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-19T16:14:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T16:14:27,950 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:27,951 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:27,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:28,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032927997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:28,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032928007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:28,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:28,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032928017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032928017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T16:14:28,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/9e4d51e1b8cb454a97e49236296d3265 2024-11-19T16:14:28,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T16:14:28,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:28,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:28,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:28,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,144 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/db29f33fa1cf4443878dca53c68afa12 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/db29f33fa1cf4443878dca53c68afa12 2024-11-19T16:14:28,159 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/A of c20cca7a0cf0913bda0c7ed239b24e88 into db29f33fa1cf4443878dca53c68afa12(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:28,159 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:28,159 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/A, priority=13, startTime=1732032867600; duration=0sec 2024-11-19T16:14:28,160 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:28,160 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:A 2024-11-19T16:14:28,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/f6d9777803d2445da5ccff026f5dada0 is 50, key is test_row_0/B:col10/1732032867581/Put/seqid=0 2024-11-19T16:14:28,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741880_1056 (size=12301) 2024-11-19T16:14:28,196 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/c12154bb191b48dd96edf85efa816c9b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/c12154bb191b48dd96edf85efa816c9b 2024-11-19T16:14:28,207 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/C of c20cca7a0cf0913bda0c7ed239b24e88 into c12154bb191b48dd96edf85efa816c9b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:28,207 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:28,207 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/C, priority=13, startTime=1732032867603; duration=0sec 2024-11-19T16:14:28,208 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:28,208 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:C 2024-11-19T16:14:28,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T16:14:28,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:28,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:28,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:28,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T16:14:28,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032928309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:28,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032928317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:28,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032928327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:28,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032928328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:28,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T16:14:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:28,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032928558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:28,563 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_0' in c20cca7a0cf0913bda0c7ed239b24e88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:14:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:28,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T16:14:28,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:28,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:28,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T16:14:28,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/f6d9777803d2445da5ccff026f5dada0 2024-11-19T16:14:28,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/726c232a2d8845eab064357529e9e9a0 is 50, key is test_row_0/C:col10/1732032867581/Put/seqid=0 2024-11-19T16:14:28,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741881_1057 (size=12301) 2024-11-19T16:14:28,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T16:14:28,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:28,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:28,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:28,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032928827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:28,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032928840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:28,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032928846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032928847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:28,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T16:14:28,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:28,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:28,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:28,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:29,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/726c232a2d8845eab064357529e9e9a0 2024-11-19T16:14:29,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/9e4d51e1b8cb454a97e49236296d3265 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9e4d51e1b8cb454a97e49236296d3265 2024-11-19T16:14:29,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9e4d51e1b8cb454a97e49236296d3265, entries=200, sequenceid=414, filesize=14.4 K 2024-11-19T16:14:29,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/f6d9777803d2445da5ccff026f5dada0 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/f6d9777803d2445da5ccff026f5dada0 2024-11-19T16:14:29,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/f6d9777803d2445da5ccff026f5dada0, entries=150, sequenceid=414, filesize=12.0 K 2024-11-19T16:14:29,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/726c232a2d8845eab064357529e9e9a0 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/726c232a2d8845eab064357529e9e9a0 2024-11-19T16:14:29,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T16:14:29,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:29,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:29,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:29,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:29,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:29,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/726c232a2d8845eab064357529e9e9a0, entries=150, sequenceid=414, filesize=12.0 K 2024-11-19T16:14:29,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~240.80 KB/246576, currentSize=93.93 KB/96180 for c20cca7a0cf0913bda0c7ed239b24e88 in 1452ms, sequenceid=414, compaction requested=false 2024-11-19T16:14:29,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T16:14:29,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T16:14:29,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:29,201 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T16:14:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:29,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/9984443b7f6440e399e7954614ce925d is 50, key is test_row_0/A:col10/1732032867656/Put/seqid=0 2024-11-19T16:14:29,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741882_1058 (size=12301) 2024-11-19T16:14:29,630 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/9984443b7f6440e399e7954614ce925d 2024-11-19T16:14:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/b99b2b00b0134c7a973a20d971d213dc is 50, key is test_row_0/B:col10/1732032867656/Put/seqid=0 2024-11-19T16:14:29,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741883_1059 (size=12301) 2024-11-19T16:14:29,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:29,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032929871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:29,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032929872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032929872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032929875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:29,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:29,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:29,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032929986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032929986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032929987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032929987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,051 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/b99b2b00b0134c7a973a20d971d213dc 2024-11-19T16:14:30,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/1a7ca5b1c9c1446fad61f9fc38c81b45 is 50, key is test_row_0/C:col10/1732032867656/Put/seqid=0 2024-11-19T16:14:30,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741884_1060 (size=12301) 2024-11-19T16:14:30,083 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/1a7ca5b1c9c1446fad61f9fc38c81b45 2024-11-19T16:14:30,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/9984443b7f6440e399e7954614ce925d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9984443b7f6440e399e7954614ce925d 2024-11-19T16:14:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T16:14:30,104 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9984443b7f6440e399e7954614ce925d, entries=150, sequenceid=434, filesize=12.0 K 2024-11-19T16:14:30,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/b99b2b00b0134c7a973a20d971d213dc as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b99b2b00b0134c7a973a20d971d213dc 2024-11-19T16:14:30,125 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b99b2b00b0134c7a973a20d971d213dc, entries=150, sequenceid=434, filesize=12.0 K 2024-11-19T16:14:30,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/1a7ca5b1c9c1446fad61f9fc38c81b45 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1a7ca5b1c9c1446fad61f9fc38c81b45 2024-11-19T16:14:30,134 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1a7ca5b1c9c1446fad61f9fc38c81b45, entries=150, sequenceid=434, filesize=12.0 K 2024-11-19T16:14:30,137 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c20cca7a0cf0913bda0c7ed239b24e88 in 935ms, sequenceid=434, compaction requested=true 2024-11-19T16:14:30,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:30,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:30,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-19T16:14:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-19T16:14:30,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-19T16:14:30,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1880 sec 2024-11-19T16:14:30,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.1940 sec 2024-11-19T16:14:30,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T16:14:30,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:30,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:30,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:30,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:30,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:30,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:30,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/61b9956f9f004a75aef4274ac4484ee6 is 50, key is test_row_0/A:col10/1732032869871/Put/seqid=0 2024-11-19T16:14:30,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741885_1061 (size=12301) 2024-11-19T16:14:30,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/61b9956f9f004a75aef4274ac4484ee6 2024-11-19T16:14:30,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032930225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032930226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/9b9860d023514186bbfde9af44a400e8 is 50, key is test_row_0/B:col10/1732032869871/Put/seqid=0 2024-11-19T16:14:30,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032930234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032930234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741886_1062 (size=12301) 2024-11-19T16:14:30,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032930347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032930347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032930347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032930347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032930556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032930556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032930557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032930567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032930576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,578 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_0' in c20cca7a0cf0913bda0c7ed239b24e88 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:14:30,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/9b9860d023514186bbfde9af44a400e8 2024-11-19T16:14:30,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/655a70e7649a466d98f8514f7597dafa is 50, key is test_row_0/C:col10/1732032869871/Put/seqid=0 2024-11-19T16:14:30,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741887_1063 (size=12301) 2024-11-19T16:14:30,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032930867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032930867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032930867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:30,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032930876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:31,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/655a70e7649a466d98f8514f7597dafa 2024-11-19T16:14:31,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/61b9956f9f004a75aef4274ac4484ee6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/61b9956f9f004a75aef4274ac4484ee6 2024-11-19T16:14:31,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/61b9956f9f004a75aef4274ac4484ee6, entries=150, sequenceid=451, filesize=12.0 K 2024-11-19T16:14:31,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/9b9860d023514186bbfde9af44a400e8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/9b9860d023514186bbfde9af44a400e8 2024-11-19T16:14:31,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/9b9860d023514186bbfde9af44a400e8, entries=150, sequenceid=451, filesize=12.0 K 2024-11-19T16:14:31,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/655a70e7649a466d98f8514f7597dafa as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/655a70e7649a466d98f8514f7597dafa 2024-11-19T16:14:31,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/655a70e7649a466d98f8514f7597dafa, entries=150, sequenceid=451, filesize=12.0 K 2024-11-19T16:14:31,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for c20cca7a0cf0913bda0c7ed239b24e88 in 910ms, sequenceid=451, compaction requested=true 2024-11-19T16:14:31,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:31,111 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:31,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:31,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:31,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:31,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:31,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:31,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T16:14:31,112 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:31,113 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52088 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:31,113 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/A is initiating minor compaction (all files) 2024-11-19T16:14:31,114 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/A in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:31,114 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/db29f33fa1cf4443878dca53c68afa12, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9e4d51e1b8cb454a97e49236296d3265, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9984443b7f6440e399e7954614ce925d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/61b9956f9f004a75aef4274ac4484ee6] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=50.9 K 2024-11-19T16:14:31,115 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:31,115 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting db29f33fa1cf4443878dca53c68afa12, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732032866358 2024-11-19T16:14:31,115 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/B is initiating minor compaction (all files) 2024-11-19T16:14:31,115 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/B in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:31,115 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d3a2c791aed448538d5cd98093a73607, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/f6d9777803d2445da5ccff026f5dada0, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b99b2b00b0134c7a973a20d971d213dc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/9b9860d023514186bbfde9af44a400e8] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=48.5 K 2024-11-19T16:14:31,115 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e4d51e1b8cb454a97e49236296d3265, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732032867581 2024-11-19T16:14:31,115 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3a2c791aed448538d5cd98093a73607, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732032866358 2024-11-19T16:14:31,116 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 9984443b7f6440e399e7954614ce925d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732032867656 2024-11-19T16:14:31,116 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6d9777803d2445da5ccff026f5dada0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732032867581 2024-11-19T16:14:31,116 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 61b9956f9f004a75aef4274ac4484ee6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732032869871 2024-11-19T16:14:31,117 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting b99b2b00b0134c7a973a20d971d213dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732032867656 2024-11-19T16:14:31,117 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b9860d023514186bbfde9af44a400e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732032869871 2024-11-19T16:14:31,129 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:31,130 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/bf7567d6eb154287bb03ef424305a482 is 50, key is test_row_0/A:col10/1732032869871/Put/seqid=0 2024-11-19T16:14:31,130 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#B#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:31,131 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/056c5f7154ab42f095a4545c2c02fd81 is 50, key is test_row_0/B:col10/1732032869871/Put/seqid=0 2024-11-19T16:14:31,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741888_1064 (size=12881) 2024-11-19T16:14:31,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741889_1065 (size=12881) 2024-11-19T16:14:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:31,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T16:14:31,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:31,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:31,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:31,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:31,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:31,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:31,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/5bf8781cf1c94392a6236a768fee31b8 is 50, key is test_row_0/A:col10/1732032870231/Put/seqid=0 2024-11-19T16:14:31,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741890_1066 (size=14741) 2024-11-19T16:14:31,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/5bf8781cf1c94392a6236a768fee31b8 2024-11-19T16:14:31,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032931398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:31,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:31,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/3ff9090a6b27427e96bf14d259ad1143 is 50, key is test_row_0/B:col10/1732032870231/Put/seqid=0 2024-11-19T16:14:31,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032931404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032931404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:31,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032931405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:31,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741891_1067 (size=12301) 2024-11-19T16:14:31,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032931517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:31,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032931517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:31,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032931517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032931517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:31,560 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/056c5f7154ab42f095a4545c2c02fd81 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/056c5f7154ab42f095a4545c2c02fd81 2024-11-19T16:14:31,563 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/bf7567d6eb154287bb03ef424305a482 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/bf7567d6eb154287bb03ef424305a482 2024-11-19T16:14:31,571 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/A of c20cca7a0cf0913bda0c7ed239b24e88 into bf7567d6eb154287bb03ef424305a482(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:31,571 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:31,571 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/A, priority=12, startTime=1732032871111; duration=0sec 2024-11-19T16:14:31,572 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:31,572 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:A 2024-11-19T16:14:31,572 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:31,572 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/B of c20cca7a0cf0913bda0c7ed239b24e88 into 056c5f7154ab42f095a4545c2c02fd81(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:31,572 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:31,572 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/B, priority=12, startTime=1732032871111; duration=0sec 2024-11-19T16:14:31,573 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:31,573 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:B 2024-11-19T16:14:31,574 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:31,574 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/C is initiating minor compaction (all files) 2024-11-19T16:14:31,574 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/C in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:31,574 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/c12154bb191b48dd96edf85efa816c9b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/726c232a2d8845eab064357529e9e9a0, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1a7ca5b1c9c1446fad61f9fc38c81b45, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/655a70e7649a466d98f8514f7597dafa] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=48.5 K 2024-11-19T16:14:31,575 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting c12154bb191b48dd96edf85efa816c9b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732032866358 2024-11-19T16:14:31,577 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 726c232a2d8845eab064357529e9e9a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732032867581 2024-11-19T16:14:31,578 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a7ca5b1c9c1446fad61f9fc38c81b45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732032867656 2024-11-19T16:14:31,578 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 655a70e7649a466d98f8514f7597dafa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732032869871 2024-11-19T16:14:31,591 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#C#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:31,591 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/1260b6087cff494fbb75142ca51af827 is 50, key is test_row_0/C:col10/1732032869871/Put/seqid=0 2024-11-19T16:14:31,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741892_1068 (size=12881) 2024-11-19T16:14:31,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032931727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032931727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032931727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032931727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:31,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/3ff9090a6b27427e96bf14d259ad1143 2024-11-19T16:14:31,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/7c6c2b9d7f2c490fbbb8c31f46791682 is 50, key is test_row_0/C:col10/1732032870231/Put/seqid=0 2024-11-19T16:14:31,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741893_1069 (size=12301) 2024-11-19T16:14:32,009 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/1260b6087cff494fbb75142ca51af827 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1260b6087cff494fbb75142ca51af827 2024-11-19T16:14:32,016 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/C of c20cca7a0cf0913bda0c7ed239b24e88 into 1260b6087cff494fbb75142ca51af827(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:32,016 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:32,016 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/C, priority=12, startTime=1732032871111; duration=0sec 2024-11-19T16:14:32,017 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:32,017 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:C 2024-11-19T16:14:32,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032932037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032932037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032932037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032932037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T16:14:32,127 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:32,128 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:32,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-19T16:14:32,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T16:14:32,130 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:32,131 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:32,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:32,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/7c6c2b9d7f2c490fbbb8c31f46791682 2024-11-19T16:14:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T16:14:32,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/5bf8781cf1c94392a6236a768fee31b8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/5bf8781cf1c94392a6236a768fee31b8 2024-11-19T16:14:32,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/5bf8781cf1c94392a6236a768fee31b8, entries=200, sequenceid=468, filesize=14.4 K 2024-11-19T16:14:32,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/3ff9090a6b27427e96bf14d259ad1143 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/3ff9090a6b27427e96bf14d259ad1143 2024-11-19T16:14:32,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/3ff9090a6b27427e96bf14d259ad1143, entries=150, sequenceid=468, filesize=12.0 K 2024-11-19T16:14:32,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/7c6c2b9d7f2c490fbbb8c31f46791682 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/7c6c2b9d7f2c490fbbb8c31f46791682 2024-11-19T16:14:32,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/7c6c2b9d7f2c490fbbb8c31f46791682, entries=150, sequenceid=468, filesize=12.0 K 2024-11-19T16:14:32,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c20cca7a0cf0913bda0c7ed239b24e88 in 888ms, sequenceid=468, compaction requested=false 2024-11-19T16:14:32,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:32,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T16:14:32,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:32,285 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T16:14:32,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:32,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:32,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:32,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:32,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:32,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:32,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/f10b00c76f3047dfa7bfe2601e5d4aeb is 50, key is test_row_0/A:col10/1732032871403/Put/seqid=0 2024-11-19T16:14:32,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741894_1070 (size=12301) 2024-11-19T16:14:32,303 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/f10b00c76f3047dfa7bfe2601e5d4aeb 2024-11-19T16:14:32,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/b4fe97e0606e4956a862b45de6b68656 is 50, key is test_row_0/B:col10/1732032871403/Put/seqid=0 2024-11-19T16:14:32,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741895_1071 (size=12301) 2024-11-19T16:14:32,325 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/b4fe97e0606e4956a862b45de6b68656 2024-11-19T16:14:32,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/ab0793503d2f45dd872911cae83e0595 is 50, key is test_row_0/C:col10/1732032871403/Put/seqid=0 2024-11-19T16:14:32,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741896_1072 (size=12301) 2024-11-19T16:14:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T16:14:32,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:32,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:32,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032932566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032932569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032932569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032932571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032932676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032932687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032932687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032932687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T16:14:32,757 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/ab0793503d2f45dd872911cae83e0595 2024-11-19T16:14:32,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/f10b00c76f3047dfa7bfe2601e5d4aeb as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/f10b00c76f3047dfa7bfe2601e5d4aeb 2024-11-19T16:14:32,770 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/f10b00c76f3047dfa7bfe2601e5d4aeb, entries=150, sequenceid=486, filesize=12.0 K 2024-11-19T16:14:32,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/b4fe97e0606e4956a862b45de6b68656 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b4fe97e0606e4956a862b45de6b68656 2024-11-19T16:14:32,779 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b4fe97e0606e4956a862b45de6b68656, entries=150, sequenceid=486, filesize=12.0 K 2024-11-19T16:14:32,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/ab0793503d2f45dd872911cae83e0595 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/ab0793503d2f45dd872911cae83e0595 2024-11-19T16:14:32,786 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/ab0793503d2f45dd872911cae83e0595, entries=150, sequenceid=486, filesize=12.0 K 2024-11-19T16:14:32,787 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c20cca7a0cf0913bda0c7ed239b24e88 in 503ms, sequenceid=486, compaction requested=true 2024-11-19T16:14:32,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-19T16:14:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-19T16:14:32,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-19T16:14:32,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 658 msec 2024-11-19T16:14:32,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 664 msec 2024-11-19T16:14:32,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T16:14:32,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:32,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:32,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:32,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:32,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:32,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:32,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/3b2340a630d948478cbd67f907d6e7ab is 50, key is test_row_0/A:col10/1732032872564/Put/seqid=0 2024-11-19T16:14:32,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032932911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032932911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741897_1073 (size=14741) 2024-11-19T16:14:32,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032932923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032932923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:33,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032933026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:33,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:33,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032933026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:33,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032933037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:33,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032933037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:33,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032933237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032933237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:33,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:33,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032933247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:33,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:33,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032933247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T16:14:33,267 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:33,267 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:33,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-19T16:14:33,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T16:14:33,269 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:33,270 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:33,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:33,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/3b2340a630d948478cbd67f907d6e7ab 2024-11-19T16:14:33,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/fedff7b5b0f74106b354e56409c930d1 is 50, key is test_row_0/B:col10/1732032872564/Put/seqid=0 2024-11-19T16:14:33,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741898_1074 (size=12301) 2024-11-19T16:14:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T16:14:33,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T16:14:33,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:33,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:33,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:33,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:33,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032933547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:33,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032933547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:33,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032933556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:33,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032933556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:33,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T16:14:33,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:33,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:33,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:33,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:33,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:33,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T16:14:33,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T16:14:33,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:33,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:33,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:33,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:33,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:33,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/fedff7b5b0f74106b354e56409c930d1 2024-11-19T16:14:33,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/a02a81bfb70642e09cc3c5eac4ebd349 is 50, key is test_row_0/C:col10/1732032872564/Put/seqid=0 2024-11-19T16:14:33,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741899_1075 (size=12301) 2024-11-19T16:14:33,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T16:14:33,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:33,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:33,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:33,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:33,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:33,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T16:14:34,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T16:14:34,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:34,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:34,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032934056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032934057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:34,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032934067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032934067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:34,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/a02a81bfb70642e09cc3c5eac4ebd349 2024-11-19T16:14:34,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/3b2340a630d948478cbd67f907d6e7ab as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/3b2340a630d948478cbd67f907d6e7ab 2024-11-19T16:14:34,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/3b2340a630d948478cbd67f907d6e7ab, entries=200, sequenceid=504, filesize=14.4 K 2024-11-19T16:14:34,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/fedff7b5b0f74106b354e56409c930d1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/fedff7b5b0f74106b354e56409c930d1 2024-11-19T16:14:34,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/fedff7b5b0f74106b354e56409c930d1, entries=150, sequenceid=504, filesize=12.0 K 2024-11-19T16:14:34,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/a02a81bfb70642e09cc3c5eac4ebd349 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a02a81bfb70642e09cc3c5eac4ebd349 2024-11-19T16:14:34,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a02a81bfb70642e09cc3c5eac4ebd349, entries=150, sequenceid=504, filesize=12.0 K 2024-11-19T16:14:34,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c20cca7a0cf0913bda0c7ed239b24e88 in 1293ms, sequenceid=504, compaction requested=true 2024-11-19T16:14:34,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:34,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:34,183 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:34,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:34,183 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:34,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:34,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:34,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:34,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:34,184 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54664 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:34,184 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:34,184 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/A is initiating minor compaction (all files) 2024-11-19T16:14:34,184 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/B is initiating minor compaction (all files) 2024-11-19T16:14:34,184 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/A in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:34,184 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/B in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:34,185 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/bf7567d6eb154287bb03ef424305a482, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/5bf8781cf1c94392a6236a768fee31b8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/f10b00c76f3047dfa7bfe2601e5d4aeb, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/3b2340a630d948478cbd67f907d6e7ab] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=53.4 K 2024-11-19T16:14:34,185 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/056c5f7154ab42f095a4545c2c02fd81, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/3ff9090a6b27427e96bf14d259ad1143, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b4fe97e0606e4956a862b45de6b68656, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/fedff7b5b0f74106b354e56409c930d1] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=48.6 K 2024-11-19T16:14:34,185 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf7567d6eb154287bb03ef424305a482, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732032869871 2024-11-19T16:14:34,185 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 056c5f7154ab42f095a4545c2c02fd81, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732032869871 2024-11-19T16:14:34,185 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bf8781cf1c94392a6236a768fee31b8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1732032870220 2024-11-19T16:14:34,185 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ff9090a6b27427e96bf14d259ad1143, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1732032870231 2024-11-19T16:14:34,185 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting f10b00c76f3047dfa7bfe2601e5d4aeb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732032871390 2024-11-19T16:14:34,186 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting b4fe97e0606e4956a862b45de6b68656, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732032871390 2024-11-19T16:14:34,186 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b2340a630d948478cbd67f907d6e7ab, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732032872564 2024-11-19T16:14:34,186 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting fedff7b5b0f74106b354e56409c930d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732032872564 2024-11-19T16:14:34,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T16:14:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:34,189 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T16:14:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:34,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:34,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:34,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:34,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/48117146fd9548d689d01144b07d32d4 is 50, key is test_row_0/A:col10/1732032872920/Put/seqid=0 2024-11-19T16:14:34,197 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:34,198 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/8179c7baed5e417a8e307d03ee30a8b7 is 50, key is test_row_0/B:col10/1732032872564/Put/seqid=0 2024-11-19T16:14:34,201 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#A#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:34,201 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/4ee0e9ae124a4c1c80e3d62dae70294e is 50, key is test_row_0/A:col10/1732032872564/Put/seqid=0 2024-11-19T16:14:34,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741900_1076 (size=12301) 2024-11-19T16:14:34,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741901_1077 (size=13017) 2024-11-19T16:14:34,238 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/8179c7baed5e417a8e307d03ee30a8b7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8179c7baed5e417a8e307d03ee30a8b7 2024-11-19T16:14:34,248 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/B of c20cca7a0cf0913bda0c7ed239b24e88 into 8179c7baed5e417a8e307d03ee30a8b7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:34,248 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:34,248 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/B, priority=12, startTime=1732032874183; duration=0sec 2024-11-19T16:14:34,248 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:34,248 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:B 2024-11-19T16:14:34,248 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:34,250 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:34,250 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/C is initiating minor compaction (all files) 2024-11-19T16:14:34,250 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/C in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:34,251 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1260b6087cff494fbb75142ca51af827, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/7c6c2b9d7f2c490fbbb8c31f46791682, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/ab0793503d2f45dd872911cae83e0595, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a02a81bfb70642e09cc3c5eac4ebd349] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=48.6 K 2024-11-19T16:14:34,251 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 1260b6087cff494fbb75142ca51af827, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732032869871 2024-11-19T16:14:34,252 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c6c2b9d7f2c490fbbb8c31f46791682, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1732032870231 2024-11-19T16:14:34,252 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting ab0793503d2f45dd872911cae83e0595, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732032871390 2024-11-19T16:14:34,252 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a02a81bfb70642e09cc3c5eac4ebd349, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732032872564 2024-11-19T16:14:34,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741902_1078 (size=13017) 2024-11-19T16:14:34,285 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#C#compaction#66 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:34,286 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/3f0d3e3abc384366aaf37b3b21ed3938 is 50, key is test_row_0/C:col10/1732032872564/Put/seqid=0 2024-11-19T16:14:34,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741903_1079 (size=13017) 2024-11-19T16:14:34,316 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/3f0d3e3abc384366aaf37b3b21ed3938 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/3f0d3e3abc384366aaf37b3b21ed3938 2024-11-19T16:14:34,326 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/C of c20cca7a0cf0913bda0c7ed239b24e88 into 3f0d3e3abc384366aaf37b3b21ed3938(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:34,326 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:34,326 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/C, priority=12, startTime=1732032874183; duration=0sec 2024-11-19T16:14:34,326 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:34,326 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:C 2024-11-19T16:14:34,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T16:14:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:34,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:34,624 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/48117146fd9548d689d01144b07d32d4 2024-11-19T16:14:34,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/ee38b75fbc854c0ba394306eacfaf5fc is 50, key is test_row_0/B:col10/1732032872920/Put/seqid=0 2024-11-19T16:14:34,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741904_1080 (size=12301) 2024-11-19T16:14:34,647 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/ee38b75fbc854c0ba394306eacfaf5fc 2024-11-19T16:14:34,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/bbc35bc68bd548f991c7a3b8078b168e is 50, key is test_row_0/C:col10/1732032872920/Put/seqid=0 2024-11-19T16:14:34,675 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/4ee0e9ae124a4c1c80e3d62dae70294e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/4ee0e9ae124a4c1c80e3d62dae70294e 2024-11-19T16:14:34,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741905_1081 (size=12301) 2024-11-19T16:14:34,681 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/bbc35bc68bd548f991c7a3b8078b168e 2024-11-19T16:14:34,689 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/A of c20cca7a0cf0913bda0c7ed239b24e88 into 4ee0e9ae124a4c1c80e3d62dae70294e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:34,689 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:34,689 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/A, priority=12, startTime=1732032874183; duration=0sec 2024-11-19T16:14:34,690 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:34,690 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:A 2024-11-19T16:14:34,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/48117146fd9548d689d01144b07d32d4 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/48117146fd9548d689d01144b07d32d4 2024-11-19T16:14:34,705 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/48117146fd9548d689d01144b07d32d4, entries=150, sequenceid=518, filesize=12.0 K 2024-11-19T16:14:34,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032934703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/ee38b75fbc854c0ba394306eacfaf5fc as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ee38b75fbc854c0ba394306eacfaf5fc 2024-11-19T16:14:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:34,721 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ee38b75fbc854c0ba394306eacfaf5fc, entries=150, sequenceid=518, filesize=12.0 K 2024-11-19T16:14:34,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/bbc35bc68bd548f991c7a3b8078b168e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/bbc35bc68bd548f991c7a3b8078b168e 2024-11-19T16:14:34,730 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/bbc35bc68bd548f991c7a3b8078b168e, entries=150, sequenceid=518, filesize=12.0 K 2024-11-19T16:14:34,732 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c20cca7a0cf0913bda0c7ed239b24e88 in 543ms, sequenceid=518, compaction requested=false 2024-11-19T16:14:34,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:34,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:34,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-19T16:14:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-19T16:14:34,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-19T16:14:34,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4650 sec 2024-11-19T16:14:34,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4720 sec 2024-11-19T16:14:34,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T16:14:34,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:34,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:34,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:34,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:34,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:34,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:34,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/0aca97709d1b4f89b72cf5c05bac735b is 50, key is test_row_0/A:col10/1732032874818/Put/seqid=0 2024-11-19T16:14:34,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741906_1082 (size=14741) 2024-11-19T16:14:34,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/0aca97709d1b4f89b72cf5c05bac735b 2024-11-19T16:14:34,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032934866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:34,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:34,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/08a3673a52ad4056a1861b72936f5176 is 50, key is test_row_0/B:col10/1732032874818/Put/seqid=0 2024-11-19T16:14:34,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741907_1083 (size=12301) 2024-11-19T16:14:34,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/08a3673a52ad4056a1861b72936f5176 2024-11-19T16:14:34,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/0dcfb3ef93a6470d86ce8c8d760689b8 is 50, key is test_row_0/C:col10/1732032874818/Put/seqid=0 2024-11-19T16:14:34,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741908_1084 (size=12301) 2024-11-19T16:14:34,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032934977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:35,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032935078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:35,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:35,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032935078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032935078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:35,080 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in c20cca7a0cf0913bda0c7ed239b24e88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:14:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:35,083 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in c20cca7a0cf0913bda0c7ed239b24e88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:14:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:35,084 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in c20cca7a0cf0913bda0c7ed239b24e88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:14:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:35,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032935087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:35,092 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_0' in c20cca7a0cf0913bda0c7ed239b24e88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:14:35,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:35,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032935187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:35,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/0dcfb3ef93a6470d86ce8c8d760689b8 2024-11-19T16:14:35,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/0aca97709d1b4f89b72cf5c05bac735b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/0aca97709d1b4f89b72cf5c05bac735b 2024-11-19T16:14:35,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/0aca97709d1b4f89b72cf5c05bac735b, entries=200, sequenceid=540, filesize=14.4 K 2024-11-19T16:14:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/08a3673a52ad4056a1861b72936f5176 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/08a3673a52ad4056a1861b72936f5176 2024-11-19T16:14:35,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/08a3673a52ad4056a1861b72936f5176, entries=150, sequenceid=540, filesize=12.0 K 2024-11-19T16:14:35,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/0dcfb3ef93a6470d86ce8c8d760689b8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/0dcfb3ef93a6470d86ce8c8d760689b8 2024-11-19T16:14:35,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/0dcfb3ef93a6470d86ce8c8d760689b8, entries=150, sequenceid=540, filesize=12.0 K 2024-11-19T16:14:35,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c20cca7a0cf0913bda0c7ed239b24e88 in 548ms, sequenceid=540, compaction requested=true 2024-11-19T16:14:35,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:35,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:35,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:35,369 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:35,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:35,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:35,369 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:35,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:35,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:35,371 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:35,371 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:35,371 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/A is initiating minor compaction (all files) 2024-11-19T16:14:35,371 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/B is initiating minor compaction (all files) 2024-11-19T16:14:35,371 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/A in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:35,371 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/4ee0e9ae124a4c1c80e3d62dae70294e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/48117146fd9548d689d01144b07d32d4, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/0aca97709d1b4f89b72cf5c05bac735b] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=39.1 K 2024-11-19T16:14:35,371 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/B in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:35,372 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8179c7baed5e417a8e307d03ee30a8b7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ee38b75fbc854c0ba394306eacfaf5fc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/08a3673a52ad4056a1861b72936f5176] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=36.7 K 2024-11-19T16:14:35,372 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ee0e9ae124a4c1c80e3d62dae70294e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732032872564 2024-11-19T16:14:35,372 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 8179c7baed5e417a8e307d03ee30a8b7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732032872564 2024-11-19T16:14:35,372 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48117146fd9548d689d01144b07d32d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732032872908 2024-11-19T16:14:35,373 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting ee38b75fbc854c0ba394306eacfaf5fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732032872908 2024-11-19T16:14:35,373 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 08a3673a52ad4056a1861b72936f5176, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732032874683 2024-11-19T16:14:35,374 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aca97709d1b4f89b72cf5c05bac735b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732032874683 2024-11-19T16:14:35,383 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#B#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:35,384 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/36114f21d6714216a1ea073f01840ff4 is 50, key is test_row_0/B:col10/1732032874818/Put/seqid=0 2024-11-19T16:14:35,386 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:35,387 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/00d72f4408034854a6d8f4c3fcf014cc is 50, key is test_row_0/A:col10/1732032874818/Put/seqid=0 2024-11-19T16:14:35,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741909_1085 (size=13119) 2024-11-19T16:14:35,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741910_1086 (size=13119) 2024-11-19T16:14:35,406 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/00d72f4408034854a6d8f4c3fcf014cc as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/00d72f4408034854a6d8f4c3fcf014cc 2024-11-19T16:14:35,413 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/A of c20cca7a0cf0913bda0c7ed239b24e88 into 00d72f4408034854a6d8f4c3fcf014cc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:35,413 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:35,413 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/A, priority=13, startTime=1732032875369; duration=0sec 2024-11-19T16:14:35,413 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:35,413 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:A 2024-11-19T16:14:35,413 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:35,415 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:35,415 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/C is initiating minor compaction (all files) 2024-11-19T16:14:35,415 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/C in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:35,415 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/3f0d3e3abc384366aaf37b3b21ed3938, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/bbc35bc68bd548f991c7a3b8078b168e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/0dcfb3ef93a6470d86ce8c8d760689b8] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=36.7 K 2024-11-19T16:14:35,416 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f0d3e3abc384366aaf37b3b21ed3938, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732032872564 2024-11-19T16:14:35,416 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbc35bc68bd548f991c7a3b8078b168e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732032872908 2024-11-19T16:14:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T16:14:35,417 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:35,417 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-19T16:14:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T16:14:35,420 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:35,421 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:35,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:35,427 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dcfb3ef93a6470d86ce8c8d760689b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732032874683 2024-11-19T16:14:35,441 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#C#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:35,442 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/23da7105cf5b4a9ba4f0d468887c20bb is 50, key is test_row_0/C:col10/1732032874818/Put/seqid=0 2024-11-19T16:14:35,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741911_1087 (size=13119) 2024-11-19T16:14:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:35,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T16:14:35,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:35,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:35,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:35,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:35,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:35,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:35,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/417cce7605894c77b399c176ae950da9 is 50, key is test_row_0/A:col10/1732032874860/Put/seqid=0 2024-11-19T16:14:35,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741912_1088 (size=9857) 2024-11-19T16:14:35,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/417cce7605894c77b399c176ae950da9 2024-11-19T16:14:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T16:14:35,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/15eacfe1aae040d3979528032a7a2c40 is 50, key is test_row_0/B:col10/1732032874860/Put/seqid=0 2024-11-19T16:14:35,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741913_1089 (size=9857) 2024-11-19T16:14:35,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032935569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:35,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T16:14:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:35,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:35,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032935676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:35,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T16:14:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:35,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T16:14:35,800 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/36114f21d6714216a1ea073f01840ff4 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/36114f21d6714216a1ea073f01840ff4 2024-11-19T16:14:35,806 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/B of c20cca7a0cf0913bda0c7ed239b24e88 into 36114f21d6714216a1ea073f01840ff4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:35,806 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:35,806 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/B, priority=13, startTime=1732032875369; duration=0sec 2024-11-19T16:14:35,806 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:35,806 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:B 2024-11-19T16:14:35,855 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/23da7105cf5b4a9ba4f0d468887c20bb as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/23da7105cf5b4a9ba4f0d468887c20bb 2024-11-19T16:14:35,861 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/C of c20cca7a0cf0913bda0c7ed239b24e88 into 23da7105cf5b4a9ba4f0d468887c20bb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:35,861 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:35,861 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/C, priority=13, startTime=1732032875369; duration=0sec 2024-11-19T16:14:35,861 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:35,861 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:C 2024-11-19T16:14:35,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T16:14:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:35,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:35,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032935887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:35,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/15eacfe1aae040d3979528032a7a2c40 2024-11-19T16:14:35,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/b09ef57c94cb4a2281c505f265e7e807 is 50, key is test_row_0/C:col10/1732032874860/Put/seqid=0 2024-11-19T16:14:35,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741914_1090 (size=9857) 2024-11-19T16:14:36,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T16:14:36,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:36,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:36,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:36,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:36,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T16:14:36,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T16:14:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:36,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:36,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032936197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:36,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:36,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:36,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T16:14:36,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:36,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:36,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:36,348 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37355 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=32a8860bb9d6,43095,1732032854050, table=TestAcidGuarantees, region=c20cca7a0cf0913bda0c7ed239b24e88. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T16:14:36,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/b09ef57c94cb4a2281c505f265e7e807 2024-11-19T16:14:36,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/417cce7605894c77b399c176ae950da9 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/417cce7605894c77b399c176ae950da9 2024-11-19T16:14:36,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/417cce7605894c77b399c176ae950da9, entries=100, sequenceid=555, filesize=9.6 K 2024-11-19T16:14:36,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/15eacfe1aae040d3979528032a7a2c40 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/15eacfe1aae040d3979528032a7a2c40 2024-11-19T16:14:36,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/15eacfe1aae040d3979528032a7a2c40, entries=100, sequenceid=555, filesize=9.6 K 2024-11-19T16:14:36,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/b09ef57c94cb4a2281c505f265e7e807 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/b09ef57c94cb4a2281c505f265e7e807 2024-11-19T16:14:36,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/b09ef57c94cb4a2281c505f265e7e807, entries=100, sequenceid=555, filesize=9.6 K 2024-11-19T16:14:36,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c20cca7a0cf0913bda0c7ed239b24e88 in 896ms, sequenceid=555, compaction requested=false 2024-11-19T16:14:36,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:36,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T16:14:36,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:36,499 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T16:14:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:36,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/1e3e0c63f2334da695ad8315e603c65d is 50, key is test_row_0/A:col10/1732032875554/Put/seqid=0 2024-11-19T16:14:36,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741915_1091 (size=12301) 2024-11-19T16:14:36,519 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/1e3e0c63f2334da695ad8315e603c65d 2024-11-19T16:14:36,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/50d0fc03941a4dccb5d2fead33fcb0ff is 50, key is test_row_0/B:col10/1732032875554/Put/seqid=0 2024-11-19T16:14:36,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741916_1092 (size=12301) 2024-11-19T16:14:36,544 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/50d0fc03941a4dccb5d2fead33fcb0ff 2024-11-19T16:14:36,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/0436a870b6b246d18e70c6a32789a122 is 50, key is test_row_0/C:col10/1732032875554/Put/seqid=0 2024-11-19T16:14:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T16:14:36,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741917_1093 (size=12301) 2024-11-19T16:14:36,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:36,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032936753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:36,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032936867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:36,966 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/0436a870b6b246d18e70c6a32789a122 2024-11-19T16:14:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/1e3e0c63f2334da695ad8315e603c65d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/1e3e0c63f2334da695ad8315e603c65d 2024-11-19T16:14:36,977 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/1e3e0c63f2334da695ad8315e603c65d, entries=150, sequenceid=575, filesize=12.0 K 2024-11-19T16:14:36,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/50d0fc03941a4dccb5d2fead33fcb0ff as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/50d0fc03941a4dccb5d2fead33fcb0ff 2024-11-19T16:14:36,983 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/50d0fc03941a4dccb5d2fead33fcb0ff, entries=150, sequenceid=575, filesize=12.0 K 2024-11-19T16:14:36,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/0436a870b6b246d18e70c6a32789a122 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/0436a870b6b246d18e70c6a32789a122 2024-11-19T16:14:36,991 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/0436a870b6b246d18e70c6a32789a122, entries=150, sequenceid=575, filesize=12.0 K 2024-11-19T16:14:36,994 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c20cca7a0cf0913bda0c7ed239b24e88 in 494ms, sequenceid=575, compaction requested=true 2024-11-19T16:14:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-19T16:14:36,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-19T16:14:36,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-19T16:14:36,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5730 sec 2024-11-19T16:14:37,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5810 sec 2024-11-19T16:14:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:37,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T16:14:37,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:37,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:37,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:37,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:37,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:37,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:37,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/e59caa670b184f37af6f67dbe117803b is 50, key is test_row_0/A:col10/1732032876750/Put/seqid=0 2024-11-19T16:14:37,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741918_1094 (size=14741) 2024-11-19T16:14:37,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032937136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032937137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032937137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032937145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032937146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032937257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032937257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032937257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032937257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032937257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032937466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032937466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032937467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032937467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032937467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/e59caa670b184f37af6f67dbe117803b 2024-11-19T16:14:37,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/ea6a515d56ce44158171e7ee9c610279 is 50, key is test_row_0/B:col10/1732032876750/Put/seqid=0 2024-11-19T16:14:37,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741919_1095 (size=12301) 2024-11-19T16:14:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T16:14:37,567 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:37,568 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-19T16:14:37,570 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T16:14:37,571 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:37,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T16:14:37,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T16:14:37,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:37,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:37,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:37,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:37,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:37,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032937777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032937777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032937777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032937777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032937777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:37,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T16:14:37,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:37,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:37,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:37,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:37,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T16:14:37,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/ea6a515d56ce44158171e7ee9c610279 2024-11-19T16:14:37,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/4ca1876022744d22866cca93fd0c3b9d is 50, key is test_row_0/C:col10/1732032876750/Put/seqid=0 2024-11-19T16:14:37,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741920_1096 (size=12301) 2024-11-19T16:14:37,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/4ca1876022744d22866cca93fd0c3b9d 2024-11-19T16:14:37,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/e59caa670b184f37af6f67dbe117803b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e59caa670b184f37af6f67dbe117803b 2024-11-19T16:14:37,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e59caa670b184f37af6f67dbe117803b, entries=200, sequenceid=590, filesize=14.4 K 2024-11-19T16:14:37,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/ea6a515d56ce44158171e7ee9c610279 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ea6a515d56ce44158171e7ee9c610279 2024-11-19T16:14:37,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ea6a515d56ce44158171e7ee9c610279, entries=150, sequenceid=590, filesize=12.0 K 2024-11-19T16:14:37,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/4ca1876022744d22866cca93fd0c3b9d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/4ca1876022744d22866cca93fd0c3b9d 2024-11-19T16:14:37,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/4ca1876022744d22866cca93fd0c3b9d, entries=150, sequenceid=590, filesize=12.0 K 2024-11-19T16:14:37,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c20cca7a0cf0913bda0c7ed239b24e88 in 881ms, sequenceid=590, compaction requested=true 2024-11-19T16:14:37,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:37,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:37,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:37,962 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:37,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:37,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:37,962 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:37,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:37,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:37,964 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50018 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:37,964 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/A is initiating minor compaction (all files) 2024-11-19T16:14:37,964 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/A in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:37,965 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/00d72f4408034854a6d8f4c3fcf014cc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/417cce7605894c77b399c176ae950da9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/1e3e0c63f2334da695ad8315e603c65d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e59caa670b184f37af6f67dbe117803b] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=48.8 K 2024-11-19T16:14:37,965 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47578 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:37,965 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/B is initiating minor compaction (all files) 2024-11-19T16:14:37,965 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00d72f4408034854a6d8f4c3fcf014cc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732032874683 2024-11-19T16:14:37,965 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/B in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:37,965 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/36114f21d6714216a1ea073f01840ff4, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/15eacfe1aae040d3979528032a7a2c40, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/50d0fc03941a4dccb5d2fead33fcb0ff, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ea6a515d56ce44158171e7ee9c610279] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=46.5 K 2024-11-19T16:14:37,966 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 417cce7605894c77b399c176ae950da9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732032874860 2024-11-19T16:14:37,966 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 36114f21d6714216a1ea073f01840ff4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732032874683 2024-11-19T16:14:37,966 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e3e0c63f2334da695ad8315e603c65d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1732032875550 2024-11-19T16:14:37,966 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting e59caa670b184f37af6f67dbe117803b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1732032876729 2024-11-19T16:14:37,966 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 15eacfe1aae040d3979528032a7a2c40, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732032874860 2024-11-19T16:14:37,967 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 50d0fc03941a4dccb5d2fead33fcb0ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1732032875550 2024-11-19T16:14:37,967 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting ea6a515d56ce44158171e7ee9c610279, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1732032876743 2024-11-19T16:14:37,979 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#A#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:37,979 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/c681496ea1aa4e4db677acc9a65b8378 is 50, key is test_row_0/A:col10/1732032876750/Put/seqid=0 2024-11-19T16:14:37,981 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#B#compaction#85 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:37,982 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/7de18bfbd7984ba2995a19689ab00bdf is 50, key is test_row_0/B:col10/1732032876750/Put/seqid=0 2024-11-19T16:14:37,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741921_1097 (size=13255) 2024-11-19T16:14:38,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741922_1098 (size=13255) 2024-11-19T16:14:38,010 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/7de18bfbd7984ba2995a19689ab00bdf as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/7de18bfbd7984ba2995a19689ab00bdf 2024-11-19T16:14:38,017 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/B of c20cca7a0cf0913bda0c7ed239b24e88 into 7de18bfbd7984ba2995a19689ab00bdf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:38,017 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:38,017 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/B, priority=12, startTime=1732032877962; duration=0sec 2024-11-19T16:14:38,018 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:38,018 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:B 2024-11-19T16:14:38,018 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:38,022 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47578 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:38,022 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/C is initiating minor compaction (all files) 2024-11-19T16:14:38,022 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/C in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:38,023 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/23da7105cf5b4a9ba4f0d468887c20bb, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/b09ef57c94cb4a2281c505f265e7e807, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/0436a870b6b246d18e70c6a32789a122, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/4ca1876022744d22866cca93fd0c3b9d] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=46.5 K 2024-11-19T16:14:38,023 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 23da7105cf5b4a9ba4f0d468887c20bb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732032874683 2024-11-19T16:14:38,023 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting b09ef57c94cb4a2281c505f265e7e807, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732032874860 2024-11-19T16:14:38,024 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 0436a870b6b246d18e70c6a32789a122, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1732032875550 2024-11-19T16:14:38,024 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ca1876022744d22866cca93fd0c3b9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1732032876743 2024-11-19T16:14:38,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T16:14:38,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:38,030 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T16:14:38,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:38,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:38,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:38,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:38,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:38,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:38,037 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#C#compaction#86 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:38,037 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/94f4226cdf6441b7acc2084fd4002e68 is 50, key is test_row_0/C:col10/1732032876750/Put/seqid=0 2024-11-19T16:14:38,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/fa220d6869b949409735d81c11460b4a is 50, key is test_row_0/A:col10/1732032877145/Put/seqid=0 2024-11-19T16:14:38,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741923_1099 (size=12301) 2024-11-19T16:14:38,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741924_1100 (size=13255) 2024-11-19T16:14:38,065 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/94f4226cdf6441b7acc2084fd4002e68 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/94f4226cdf6441b7acc2084fd4002e68 2024-11-19T16:14:38,072 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/C of c20cca7a0cf0913bda0c7ed239b24e88 into 94f4226cdf6441b7acc2084fd4002e68(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:38,073 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:38,073 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/C, priority=12, startTime=1732032877962; duration=0sec 2024-11-19T16:14:38,073 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:38,073 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:C 2024-11-19T16:14:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T16:14:38,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:38,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032938302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032938302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032938304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032938307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032938308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,403 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/c681496ea1aa4e4db677acc9a65b8378 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/c681496ea1aa4e4db677acc9a65b8378 2024-11-19T16:14:38,409 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/A of c20cca7a0cf0913bda0c7ed239b24e88 into c681496ea1aa4e4db677acc9a65b8378(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:38,409 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:38,409 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/A, priority=12, startTime=1732032877962; duration=0sec 2024-11-19T16:14:38,409 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:38,409 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:A 2024-11-19T16:14:38,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032938417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032938417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032938417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032938417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032938427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,455 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/fa220d6869b949409735d81c11460b4a 2024-11-19T16:14:38,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/8fb4992edf614b239cd506a743513e02 is 50, key is test_row_0/B:col10/1732032877145/Put/seqid=0 2024-11-19T16:14:38,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741925_1101 (size=12301) 2024-11-19T16:14:38,472 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/8fb4992edf614b239cd506a743513e02 2024-11-19T16:14:38,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/fded48ed1b8b400fa52ab4eec895ac40 is 50, key is test_row_0/C:col10/1732032877145/Put/seqid=0 2024-11-19T16:14:38,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741926_1102 (size=12301) 2024-11-19T16:14:38,495 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/fded48ed1b8b400fa52ab4eec895ac40 2024-11-19T16:14:38,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/fa220d6869b949409735d81c11460b4a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/fa220d6869b949409735d81c11460b4a 2024-11-19T16:14:38,509 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/fa220d6869b949409735d81c11460b4a, entries=150, sequenceid=608, filesize=12.0 K 2024-11-19T16:14:38,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/8fb4992edf614b239cd506a743513e02 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8fb4992edf614b239cd506a743513e02 2024-11-19T16:14:38,518 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8fb4992edf614b239cd506a743513e02, entries=150, sequenceid=608, filesize=12.0 K 2024-11-19T16:14:38,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/fded48ed1b8b400fa52ab4eec895ac40 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/fded48ed1b8b400fa52ab4eec895ac40 2024-11-19T16:14:38,527 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/fded48ed1b8b400fa52ab4eec895ac40, entries=150, sequenceid=608, filesize=12.0 K 2024-11-19T16:14:38,528 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c20cca7a0cf0913bda0c7ed239b24e88 in 498ms, sequenceid=608, compaction requested=false 2024-11-19T16:14:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-19T16:14:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-19T16:14:38,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-19T16:14:38,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 960 msec 2024-11-19T16:14:38,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 967 msec 2024-11-19T16:14:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:38,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T16:14:38,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:38,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:38,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:38,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:38,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:38,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:38,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/2fb4306113c047588c67d631ff8b1323 is 50, key is test_row_0/A:col10/1732032878627/Put/seqid=0 2024-11-19T16:14:38,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741927_1103 (size=17181) 2024-11-19T16:14:38,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/2fb4306113c047588c67d631ff8b1323 2024-11-19T16:14:38,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032938642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032938643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032938645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032938648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/6d4a787b4cff4fa6838d3e8768cadc6e is 50, key is test_row_0/B:col10/1732032878627/Put/seqid=0 2024-11-19T16:14:38,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032938651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741928_1104 (size=12301) 2024-11-19T16:14:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T16:14:38,707 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:38,707 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-19T16:14:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T16:14:38,710 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:38,712 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:38,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:38,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032938757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032938757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032938758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032938759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032938767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T16:14:38,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T16:14:38,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:38,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:38,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:38,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:38,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:38,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032938967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032938967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032938968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032938969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:38,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032938987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T16:14:39,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:39,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:39,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:39,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:39,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T16:14:39,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/6d4a787b4cff4fa6838d3e8768cadc6e 2024-11-19T16:14:39,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/c3dbc314d7a146378bdf8b249e68125b is 50, key is test_row_0/C:col10/1732032878627/Put/seqid=0 2024-11-19T16:14:39,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741929_1105 (size=12301) 2024-11-19T16:14:39,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T16:14:39,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:39,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:39,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:39,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:39,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:39,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032939288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032939288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032939292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032939292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032939296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T16:14:39,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:39,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:39,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:39,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:39,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T16:14:39,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/c3dbc314d7a146378bdf8b249e68125b 2024-11-19T16:14:39,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T16:14:39,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:39,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:39,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:39,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:39,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:39,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/2fb4306113c047588c67d631ff8b1323 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/2fb4306113c047588c67d631ff8b1323 2024-11-19T16:14:39,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/2fb4306113c047588c67d631ff8b1323, entries=250, sequenceid=627, filesize=16.8 K 2024-11-19T16:14:39,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/6d4a787b4cff4fa6838d3e8768cadc6e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/6d4a787b4cff4fa6838d3e8768cadc6e 2024-11-19T16:14:39,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/6d4a787b4cff4fa6838d3e8768cadc6e, entries=150, sequenceid=627, filesize=12.0 K 2024-11-19T16:14:39,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/c3dbc314d7a146378bdf8b249e68125b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/c3dbc314d7a146378bdf8b249e68125b 2024-11-19T16:14:39,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/c3dbc314d7a146378bdf8b249e68125b, entries=150, sequenceid=627, filesize=12.0 K 2024-11-19T16:14:39,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c20cca7a0cf0913bda0c7ed239b24e88 in 896ms, sequenceid=627, compaction requested=true 2024-11-19T16:14:39,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:39,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:39,524 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:39,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:39,524 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:39,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:39,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:39,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:39,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:39,528 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:39,528 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:39,528 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/A is initiating minor compaction (all files) 2024-11-19T16:14:39,528 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/B is initiating minor compaction (all files) 2024-11-19T16:14:39,528 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/A in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:39,528 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/B in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:39,528 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/7de18bfbd7984ba2995a19689ab00bdf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8fb4992edf614b239cd506a743513e02, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/6d4a787b4cff4fa6838d3e8768cadc6e] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=37.0 K 2024-11-19T16:14:39,528 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/c681496ea1aa4e4db677acc9a65b8378, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/fa220d6869b949409735d81c11460b4a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/2fb4306113c047588c67d631ff8b1323] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=41.7 K 2024-11-19T16:14:39,529 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 7de18bfbd7984ba2995a19689ab00bdf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1732032876743 2024-11-19T16:14:39,529 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting c681496ea1aa4e4db677acc9a65b8378, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1732032876743 2024-11-19T16:14:39,529 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb4992edf614b239cd506a743513e02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1732032877136 2024-11-19T16:14:39,530 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa220d6869b949409735d81c11460b4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1732032877136 2024-11-19T16:14:39,530 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d4a787b4cff4fa6838d3e8768cadc6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1732032878306 2024-11-19T16:14:39,530 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fb4306113c047588c67d631ff8b1323, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1732032878301 2024-11-19T16:14:39,546 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:39,546 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#A#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:39,547 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/67473bfcf57f44d282e7833af94cf253 is 50, key is test_row_0/B:col10/1732032878627/Put/seqid=0 2024-11-19T16:14:39,547 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/7a9468b51a14474485aa46e9f58828b7 is 50, key is test_row_0/A:col10/1732032878627/Put/seqid=0 2024-11-19T16:14:39,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741930_1106 (size=13357) 2024-11-19T16:14:39,564 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/67473bfcf57f44d282e7833af94cf253 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/67473bfcf57f44d282e7833af94cf253 2024-11-19T16:14:39,572 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/B of c20cca7a0cf0913bda0c7ed239b24e88 into 67473bfcf57f44d282e7833af94cf253(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:39,572 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:39,572 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/B, priority=13, startTime=1732032879524; duration=0sec 2024-11-19T16:14:39,573 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:39,573 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:B 2024-11-19T16:14:39,573 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:39,575 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:39,575 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/C is initiating minor compaction (all files) 2024-11-19T16:14:39,575 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/C in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:39,575 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/94f4226cdf6441b7acc2084fd4002e68, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/fded48ed1b8b400fa52ab4eec895ac40, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/c3dbc314d7a146378bdf8b249e68125b] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=37.0 K 2024-11-19T16:14:39,576 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 94f4226cdf6441b7acc2084fd4002e68, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1732032876743 2024-11-19T16:14:39,577 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting fded48ed1b8b400fa52ab4eec895ac40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1732032877136 2024-11-19T16:14:39,577 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting c3dbc314d7a146378bdf8b249e68125b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1732032878306 2024-11-19T16:14:39,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741931_1107 (size=13357) 2024-11-19T16:14:39,595 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/7a9468b51a14474485aa46e9f58828b7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/7a9468b51a14474485aa46e9f58828b7 2024-11-19T16:14:39,604 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/A of c20cca7a0cf0913bda0c7ed239b24e88 into 7a9468b51a14474485aa46e9f58828b7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:39,604 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:39,604 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/A, priority=13, startTime=1732032879524; duration=0sec 2024-11-19T16:14:39,605 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:39,605 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:A 2024-11-19T16:14:39,613 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#C#compaction#95 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:39,613 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/a15ce2e684e34f958380268328cfdab7 is 50, key is test_row_0/C:col10/1732032878627/Put/seqid=0 2024-11-19T16:14:39,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741932_1108 (size=13357) 2024-11-19T16:14:39,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T16:14:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:39,636 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T16:14:39,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:39,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:39,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:39,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:39,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:39,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:39,641 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/a15ce2e684e34f958380268328cfdab7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a15ce2e684e34f958380268328cfdab7 2024-11-19T16:14:39,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/61971514c7744f369efebc7b2eae246c is 50, key is test_row_0/A:col10/1732032878643/Put/seqid=0 2024-11-19T16:14:39,653 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/C of c20cca7a0cf0913bda0c7ed239b24e88 into a15ce2e684e34f958380268328cfdab7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:39,653 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:39,653 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/C, priority=13, startTime=1732032879525; duration=0sec 2024-11-19T16:14:39,653 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:39,653 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:C 2024-11-19T16:14:39,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741933_1109 (size=12301) 2024-11-19T16:14:39,662 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/61971514c7744f369efebc7b2eae246c 2024-11-19T16:14:39,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/73298119610e445b95f823953ec5a651 is 50, key is test_row_0/B:col10/1732032878643/Put/seqid=0 2024-11-19T16:14:39,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741934_1110 (size=12301) 2024-11-19T16:14:39,687 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/73298119610e445b95f823953ec5a651 2024-11-19T16:14:39,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/2475176dddef4ab4b7361c916c3553eb is 50, key is test_row_0/C:col10/1732032878643/Put/seqid=0 2024-11-19T16:14:39,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741935_1111 (size=12301) 2024-11-19T16:14:39,702 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/2475176dddef4ab4b7361c916c3553eb 2024-11-19T16:14:39,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/61971514c7744f369efebc7b2eae246c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/61971514c7744f369efebc7b2eae246c 2024-11-19T16:14:39,714 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/61971514c7744f369efebc7b2eae246c, entries=150, sequenceid=644, filesize=12.0 K 2024-11-19T16:14:39,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/73298119610e445b95f823953ec5a651 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/73298119610e445b95f823953ec5a651 2024-11-19T16:14:39,725 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/73298119610e445b95f823953ec5a651, entries=150, sequenceid=644, filesize=12.0 K 2024-11-19T16:14:39,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/2475176dddef4ab4b7361c916c3553eb as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/2475176dddef4ab4b7361c916c3553eb 2024-11-19T16:14:39,735 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/2475176dddef4ab4b7361c916c3553eb, entries=150, sequenceid=644, filesize=12.0 K 2024-11-19T16:14:39,736 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for c20cca7a0cf0913bda0c7ed239b24e88 in 100ms, sequenceid=644, compaction requested=false 2024-11-19T16:14:39,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:39,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:39,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-19T16:14:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-19T16:14:39,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-19T16:14:39,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0250 sec 2024-11-19T16:14:39,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.0320 sec 2024-11-19T16:14:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:39,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:14:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:39,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/964cf1baa5bd449fbb9bc19aec324581 is 50, key is test_row_0/A:col10/1732032879802/Put/seqid=0 2024-11-19T16:14:39,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741936_1112 (size=12297) 2024-11-19T16:14:39,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732032879813, max timestamp=1732032879827Num uniques 859; 2024-11-19T16:14:39,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T16:14:39,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732032879813, max timestamp=1732032879827Num uniques 859; 2024-11-19T16:14:39,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T16:14:39,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732032879813, max timestamp=1732032879827Num uniques 859; 2024-11-19T16:14:39,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T16:14:39,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/964cf1baa5bd449fbb9bc19aec324581 2024-11-19T16:14:39,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/316cfdaa982347148e98b0f28e24e953 is 50, key is test_row_0/B:col10/1732032879802/Put/seqid=0 2024-11-19T16:14:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T16:14:39,847 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:39,848 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-19T16:14:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T16:14:39,850 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:39,851 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:39,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:39,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032939843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032939845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032939849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032939850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032939852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741937_1113 (size=9857) 2024-11-19T16:14:39,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/316cfdaa982347148e98b0f28e24e953 2024-11-19T16:14:39,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/46ff5b676e4846c9bfb07d6ef8723cf6 is 50, key is test_row_0/C:col10/1732032879802/Put/seqid=0 2024-11-19T16:14:39,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741938_1114 (size=9857) 2024-11-19T16:14:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T16:14:39,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032939957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032939957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032939966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032939966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032939966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:39,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:39,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T16:14:40,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:40,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:40,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:40,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:40,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:40,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T16:14:40,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:40,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:40,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:40,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:40,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T16:14:40,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032940167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032940167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032940177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032940177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032940177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/46ff5b676e4846c9bfb07d6ef8723cf6 2024-11-19T16:14:40,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/964cf1baa5bd449fbb9bc19aec324581 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/964cf1baa5bd449fbb9bc19aec324581 2024-11-19T16:14:40,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/964cf1baa5bd449fbb9bc19aec324581, entries=150, sequenceid=655, filesize=12.0 K 2024-11-19T16:14:40,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/316cfdaa982347148e98b0f28e24e953 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/316cfdaa982347148e98b0f28e24e953 2024-11-19T16:14:40,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/316cfdaa982347148e98b0f28e24e953, entries=100, sequenceid=655, filesize=9.6 K 2024-11-19T16:14:40,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/46ff5b676e4846c9bfb07d6ef8723cf6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/46ff5b676e4846c9bfb07d6ef8723cf6 2024-11-19T16:14:40,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T16:14:40,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:40,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:40,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:40,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:40,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:40,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/46ff5b676e4846c9bfb07d6ef8723cf6, entries=100, sequenceid=655, filesize=9.6 K 2024-11-19T16:14:40,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for c20cca7a0cf0913bda0c7ed239b24e88 in 509ms, sequenceid=655, compaction requested=true 2024-11-19T16:14:40,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:40,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:40,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:40,314 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:40,314 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:40,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:40,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:40,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:40,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:40,317 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37955 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:40,317 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/A is initiating minor compaction (all files) 2024-11-19T16:14:40,317 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/A in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:40,317 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/7a9468b51a14474485aa46e9f58828b7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/61971514c7744f369efebc7b2eae246c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/964cf1baa5bd449fbb9bc19aec324581] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=37.1 K 2024-11-19T16:14:40,317 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:40,317 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/B is initiating minor compaction (all files) 2024-11-19T16:14:40,317 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/B in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:40,318 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/67473bfcf57f44d282e7833af94cf253, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/73298119610e445b95f823953ec5a651, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/316cfdaa982347148e98b0f28e24e953] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=34.7 K 2024-11-19T16:14:40,318 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a9468b51a14474485aa46e9f58828b7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1732032878306 2024-11-19T16:14:40,318 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 67473bfcf57f44d282e7833af94cf253, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1732032878306 2024-11-19T16:14:40,319 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 73298119610e445b95f823953ec5a651, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=644, earliestPutTs=1732032878636 2024-11-19T16:14:40,319 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61971514c7744f369efebc7b2eae246c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=644, earliestPutTs=1732032878636 2024-11-19T16:14:40,319 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 316cfdaa982347148e98b0f28e24e953, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1732032879802 2024-11-19T16:14:40,319 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 964cf1baa5bd449fbb9bc19aec324581, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1732032879802 2024-11-19T16:14:40,342 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#B#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:40,343 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/c71c3c20a062460088fd6c1c3442a67f is 50, key is test_row_0/B:col10/1732032879802/Put/seqid=0 2024-11-19T16:14:40,350 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#A#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:40,350 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/44e2f260e9bc49e9af459079a5f50c10 is 50, key is test_row_0/A:col10/1732032879802/Put/seqid=0 2024-11-19T16:14:40,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741939_1115 (size=13459) 2024-11-19T16:14:40,365 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/c71c3c20a062460088fd6c1c3442a67f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/c71c3c20a062460088fd6c1c3442a67f 2024-11-19T16:14:40,372 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/B of c20cca7a0cf0913bda0c7ed239b24e88 into c71c3c20a062460088fd6c1c3442a67f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:40,372 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:40,372 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/B, priority=13, startTime=1732032880314; duration=0sec 2024-11-19T16:14:40,372 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:40,372 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:B 2024-11-19T16:14:40,373 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:40,374 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:40,375 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): c20cca7a0cf0913bda0c7ed239b24e88/C is initiating minor compaction (all files) 2024-11-19T16:14:40,375 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c20cca7a0cf0913bda0c7ed239b24e88/C in TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:40,375 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a15ce2e684e34f958380268328cfdab7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/2475176dddef4ab4b7361c916c3553eb, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/46ff5b676e4846c9bfb07d6ef8723cf6] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp, totalSize=34.7 K 2024-11-19T16:14:40,375 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a15ce2e684e34f958380268328cfdab7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1732032878306 2024-11-19T16:14:40,376 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 2475176dddef4ab4b7361c916c3553eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=644, earliestPutTs=1732032878636 2024-11-19T16:14:40,376 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 46ff5b676e4846c9bfb07d6ef8723cf6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1732032879802 2024-11-19T16:14:40,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741940_1116 (size=13459) 2024-11-19T16:14:40,385 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/44e2f260e9bc49e9af459079a5f50c10 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/44e2f260e9bc49e9af459079a5f50c10 2024-11-19T16:14:40,390 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c20cca7a0cf0913bda0c7ed239b24e88#C#compaction#104 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:40,390 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/a8e6fc006a6d4582b173b25793e24bc2 is 50, key is test_row_0/C:col10/1732032879802/Put/seqid=0 2024-11-19T16:14:40,393 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/A of c20cca7a0cf0913bda0c7ed239b24e88 into 44e2f260e9bc49e9af459079a5f50c10(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:40,393 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:40,394 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/A, priority=13, startTime=1732032880314; duration=0sec 2024-11-19T16:14:40,394 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:40,394 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:A 2024-11-19T16:14:40,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741941_1117 (size=13459) 2024-11-19T16:14:40,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T16:14:40,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:40,464 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-19T16:14:40,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:40,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:40,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:40,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:40,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:40,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:40,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/7d8fa8cb20c24c5eaeaac1fbbb159417 is 50, key is test_row_0/A:col10/1732032879845/Put/seqid=0 2024-11-19T16:14:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T16:14:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:40,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. as already flushing 2024-11-19T16:14:40,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741942_1118 (size=12301) 2024-11-19T16:14:40,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032940487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032940492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032940492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032940493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032940494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032940606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032940606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032940606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032940607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,821 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/a8e6fc006a6d4582b173b25793e24bc2 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a8e6fc006a6d4582b173b25793e24bc2 2024-11-19T16:14:40,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032940816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032940816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032940817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032940817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:40,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:40,827 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c20cca7a0cf0913bda0c7ed239b24e88/C of c20cca7a0cf0913bda0c7ed239b24e88 into a8e6fc006a6d4582b173b25793e24bc2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:40,827 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:40,827 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., storeName=c20cca7a0cf0913bda0c7ed239b24e88/C, priority=13, startTime=1732032880315; duration=0sec 2024-11-19T16:14:40,827 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:40,827 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:C 2024-11-19T16:14:40,890 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/7d8fa8cb20c24c5eaeaac1fbbb159417 2024-11-19T16:14:40,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/fc819121292b4aae8299208181fdca79 is 50, key is test_row_0/B:col10/1732032879845/Put/seqid=0 2024-11-19T16:14:40,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741943_1119 (size=12301) 2024-11-19T16:14:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T16:14:41,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59826 deadline: 1732032941006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:41,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032941126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032941127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032941127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:41,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032941137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:41,303 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/fc819121292b4aae8299208181fdca79 2024-11-19T16:14:41,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/966724dbaec940cfb496c92ea25afbb4 is 50, key is test_row_0/C:col10/1732032879845/Put/seqid=0 2024-11-19T16:14:41,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741944_1120 (size=12301) 2024-11-19T16:14:41,617 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-19T16:14:41,617 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-19T16:14:41,617 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:14:41,617 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:14:41,621 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-19T16:14:41,621 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:14:41,622 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:41,622 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:41,622 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:41,622 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:41,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:14:41,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:14:41,623 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:41,623 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:41,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:14:41,629 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-19T16:14:41,629 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:14:41,629 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:41,629 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:41,629 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:14:41,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:41,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:41,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59792 deadline: 1732032941637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59812 deadline: 1732032941637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59862 deadline: 1732032941637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:41,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:41,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:41,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59852 deadline: 1732032941647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:14:41,723 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/966724dbaec940cfb496c92ea25afbb4 2024-11-19T16:14:41,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/7d8fa8cb20c24c5eaeaac1fbbb159417 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/7d8fa8cb20c24c5eaeaac1fbbb159417 2024-11-19T16:14:41,739 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/7d8fa8cb20c24c5eaeaac1fbbb159417, entries=150, sequenceid=692, filesize=12.0 K 2024-11-19T16:14:41,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/fc819121292b4aae8299208181fdca79 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/fc819121292b4aae8299208181fdca79 2024-11-19T16:14:41,747 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/fc819121292b4aae8299208181fdca79, entries=150, sequenceid=692, filesize=12.0 K 2024-11-19T16:14:41,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/966724dbaec940cfb496c92ea25afbb4 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/966724dbaec940cfb496c92ea25afbb4 2024-11-19T16:14:41,754 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/966724dbaec940cfb496c92ea25afbb4, entries=150, sequenceid=692, filesize=12.0 K 2024-11-19T16:14:41,755 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for c20cca7a0cf0913bda0c7ed239b24e88 in 1291ms, sequenceid=692, compaction requested=false 2024-11-19T16:14:41,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:41,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:41,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-19T16:14:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-19T16:14:41,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-19T16:14:41,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9050 sec 2024-11-19T16:14:41,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.9110 sec 2024-11-19T16:14:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T16:14:41,997 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:42,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T16:14:42,030 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-19T16:14:42,030 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:14:42,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:42,031 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:42,031 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:42,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:14:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:42,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/996d127c1a0a4aaca1aa989b353326fc is 50, key is test_row_0/A:col10/1732032880491/Put/seqid=0 2024-11-19T16:14:42,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741945_1121 (size=12301) 2024-11-19T16:14:42,300 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T16:14:42,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/996d127c1a0a4aaca1aa989b353326fc 2024-11-19T16:14:42,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/d489c0344674473e8a95250b70fbc672 is 50, key is test_row_0/B:col10/1732032880491/Put/seqid=0 2024-11-19T16:14:42,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741946_1122 (size=12301) 2024-11-19T16:14:42,680 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-19T16:14:42,680 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-19T16:14:42,680 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-19T16:14:42,680 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:14:42,680 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:14:42,680 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:42,680 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:42,680 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:14:42,680 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:42,680 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:42,680 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:42,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:14:42,680 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:42,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:14:42,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:14:42,681 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-19T16:14:42,681 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:14:42,681 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:42,681 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:42,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:14:42,681 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T16:14:42,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-19T16:14:42,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-19T16:14:42,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 157 2024-11-19T16:14:42,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-19T16:14:42,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-19T16:14:42,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T16:14:42,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2765 2024-11-19T16:14:42,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2834 2024-11-19T16:14:42,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T16:14:42,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1231 2024-11-19T16:14:42,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3684 rows 2024-11-19T16:14:42,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1230 2024-11-19T16:14:42,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3686 rows 2024-11-19T16:14:42,682 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T16:14:42,682 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T16:14:42,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:42,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:42,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:14:42,686 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-19T16:14:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T16:14:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T16:14:42,693 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032882693"}]},"ts":"1732032882693"} 2024-11-19T16:14:42,695 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T16:14:42,695 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T16:14:42,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T16:14:42,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c20cca7a0cf0913bda0c7ed239b24e88, UNASSIGN}] 2024-11-19T16:14:42,701 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c20cca7a0cf0913bda0c7ed239b24e88, UNASSIGN 2024-11-19T16:14:42,702 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=c20cca7a0cf0913bda0c7ed239b24e88, regionState=CLOSING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:42,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c20cca7a0cf0913bda0c7ed239b24e88, UNASSIGN because future has completed 2024-11-19T16:14:42,705 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T16:14:42,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:14:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T16:14:42,864 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:42,864 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T16:14:42,865 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing c20cca7a0cf0913bda0c7ed239b24e88, disabling compactions & flushes 2024-11-19T16:14:42,866 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:42,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/d489c0344674473e8a95250b70fbc672 2024-11-19T16:14:42,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/2225317569d3455d8ae6ccf40b13c483 is 50, key is test_row_0/C:col10/1732032880491/Put/seqid=0 2024-11-19T16:14:42,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741947_1123 (size=12301) 2024-11-19T16:14:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T16:14:43,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/2225317569d3455d8ae6ccf40b13c483 2024-11-19T16:14:43,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/996d127c1a0a4aaca1aa989b353326fc as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/996d127c1a0a4aaca1aa989b353326fc 2024-11-19T16:14:43,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/996d127c1a0a4aaca1aa989b353326fc, entries=150, sequenceid=705, filesize=12.0 K 2024-11-19T16:14:43,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/d489c0344674473e8a95250b70fbc672 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d489c0344674473e8a95250b70fbc672 2024-11-19T16:14:43,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d489c0344674473e8a95250b70fbc672, entries=150, sequenceid=705, filesize=12.0 K 2024-11-19T16:14:43,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/2225317569d3455d8ae6ccf40b13c483 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/2225317569d3455d8ae6ccf40b13c483 2024-11-19T16:14:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T16:14:43,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/2225317569d3455d8ae6ccf40b13c483, entries=150, sequenceid=705, filesize=12.0 K 2024-11-19T16:14:43,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=26.84 KB/27480 for c20cca7a0cf0913bda0c7ed239b24e88 in 1292ms, sequenceid=705, compaction requested=true 2024-11-19T16:14:43,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c20cca7a0cf0913bda0c7ed239b24e88: 2024-11-19T16:14:43,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:43,322 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:43,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:43,322 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:43,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:43,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:43,322 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. after waiting 0 ms 2024-11-19T16:14:43,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c20cca7a0cf0913bda0c7ed239b24e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:43,322 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:43,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:43,322 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. because compaction request was cancelled 2024-11-19T16:14:43,322 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. because compaction request was cancelled 2024-11-19T16:14:43,322 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:A 2024-11-19T16:14:43,322 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:B 2024-11-19T16:14:43,322 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. because compaction request was cancelled 2024-11-19T16:14:43,322 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing c20cca7a0cf0913bda0c7ed239b24e88 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T16:14:43,322 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c20cca7a0cf0913bda0c7ed239b24e88:C 2024-11-19T16:14:43,322 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=A 2024-11-19T16:14:43,322 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:43,323 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=B 2024-11-19T16:14:43,323 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:43,323 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c20cca7a0cf0913bda0c7ed239b24e88, store=C 2024-11-19T16:14:43,323 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:43,327 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/b9e6aebbfdea40f8afbf7dc18c8d91ca is 50, key is test_row_0/A:col10/1732032882680/Put/seqid=0 2024-11-19T16:14:43,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741948_1124 (size=9857) 2024-11-19T16:14:43,731 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/b9e6aebbfdea40f8afbf7dc18c8d91ca 2024-11-19T16:14:43,741 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/e262596e3a1c4e61939ca7d0721795ff is 50, key is test_row_0/B:col10/1732032882680/Put/seqid=0 2024-11-19T16:14:43,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741949_1125 (size=9857) 2024-11-19T16:14:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T16:14:44,147 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/e262596e3a1c4e61939ca7d0721795ff 2024-11-19T16:14:44,207 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/3db452fedf554a5cb5fc05d28c0e8da6 is 50, key is test_row_0/C:col10/1732032882680/Put/seqid=0 2024-11-19T16:14:44,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741950_1126 (size=9857) 2024-11-19T16:14:44,611 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/3db452fedf554a5cb5fc05d28c0e8da6 2024-11-19T16:14:44,617 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/A/b9e6aebbfdea40f8afbf7dc18c8d91ca as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/b9e6aebbfdea40f8afbf7dc18c8d91ca 2024-11-19T16:14:44,621 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/b9e6aebbfdea40f8afbf7dc18c8d91ca, entries=100, sequenceid=712, filesize=9.6 K 2024-11-19T16:14:44,622 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/B/e262596e3a1c4e61939ca7d0721795ff as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/e262596e3a1c4e61939ca7d0721795ff 2024-11-19T16:14:44,626 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/e262596e3a1c4e61939ca7d0721795ff, entries=100, sequenceid=712, filesize=9.6 K 2024-11-19T16:14:44,627 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/.tmp/C/3db452fedf554a5cb5fc05d28c0e8da6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/3db452fedf554a5cb5fc05d28c0e8da6 2024-11-19T16:14:44,632 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/3db452fedf554a5cb5fc05d28c0e8da6, entries=100, sequenceid=712, filesize=9.6 K 2024-11-19T16:14:44,633 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c20cca7a0cf0913bda0c7ed239b24e88 in 1310ms, sequenceid=712, compaction requested=true 2024-11-19T16:14:44,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9ad8199da8b14b0d9c984f5b6ff54b34, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/a2188483714c402cb11707e6393e5759, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/242903c45ccc4040bda619cd3ee07804, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/18492a5bcd8d4d2389ea0b86d2117418, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/b12403353a1f41708c80c57be394dee3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/1cfe37efff3b4d0095dffc3437189ad9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/177b318a004544d4a7dfa43a4639448f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/928355d0c14d4f379ad6872fea778d70, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/d0394ae621d84b428f2cee27c50add7a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e84a094b224545e08c78980324607366, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/eebe4b55779542df889e4073435d5b83, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/41861233edc4497d92dd3ed0bbcb8cd5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/db29f33fa1cf4443878dca53c68afa12, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e798660a544c4354bbeb4c3c212b7a61, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9e4d51e1b8cb454a97e49236296d3265, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9984443b7f6440e399e7954614ce925d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/bf7567d6eb154287bb03ef424305a482, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/61b9956f9f004a75aef4274ac4484ee6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/5bf8781cf1c94392a6236a768fee31b8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/f10b00c76f3047dfa7bfe2601e5d4aeb, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/3b2340a630d948478cbd67f907d6e7ab, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/4ee0e9ae124a4c1c80e3d62dae70294e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/48117146fd9548d689d01144b07d32d4, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/0aca97709d1b4f89b72cf5c05bac735b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/00d72f4408034854a6d8f4c3fcf014cc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/417cce7605894c77b399c176ae950da9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/1e3e0c63f2334da695ad8315e603c65d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e59caa670b184f37af6f67dbe117803b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/c681496ea1aa4e4db677acc9a65b8378, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/fa220d6869b949409735d81c11460b4a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/2fb4306113c047588c67d631ff8b1323, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/7a9468b51a14474485aa46e9f58828b7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/61971514c7744f369efebc7b2eae246c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/964cf1baa5bd449fbb9bc19aec324581] to archive 2024-11-19T16:14:44,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:14:44,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9ad8199da8b14b0d9c984f5b6ff54b34 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9ad8199da8b14b0d9c984f5b6ff54b34 2024-11-19T16:14:44,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/a2188483714c402cb11707e6393e5759 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/a2188483714c402cb11707e6393e5759 2024-11-19T16:14:44,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/242903c45ccc4040bda619cd3ee07804 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/242903c45ccc4040bda619cd3ee07804 2024-11-19T16:14:44,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/18492a5bcd8d4d2389ea0b86d2117418 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/18492a5bcd8d4d2389ea0b86d2117418 2024-11-19T16:14:44,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/b12403353a1f41708c80c57be394dee3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/b12403353a1f41708c80c57be394dee3 2024-11-19T16:14:44,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/1cfe37efff3b4d0095dffc3437189ad9 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/1cfe37efff3b4d0095dffc3437189ad9 2024-11-19T16:14:44,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/177b318a004544d4a7dfa43a4639448f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/177b318a004544d4a7dfa43a4639448f 2024-11-19T16:14:44,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/928355d0c14d4f379ad6872fea778d70 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/928355d0c14d4f379ad6872fea778d70 2024-11-19T16:14:44,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/d0394ae621d84b428f2cee27c50add7a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/d0394ae621d84b428f2cee27c50add7a 2024-11-19T16:14:44,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e84a094b224545e08c78980324607366 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e84a094b224545e08c78980324607366 2024-11-19T16:14:44,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/eebe4b55779542df889e4073435d5b83 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/eebe4b55779542df889e4073435d5b83 2024-11-19T16:14:44,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/41861233edc4497d92dd3ed0bbcb8cd5 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/41861233edc4497d92dd3ed0bbcb8cd5 2024-11-19T16:14:44,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/db29f33fa1cf4443878dca53c68afa12 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/db29f33fa1cf4443878dca53c68afa12 2024-11-19T16:14:44,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e798660a544c4354bbeb4c3c212b7a61 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e798660a544c4354bbeb4c3c212b7a61 2024-11-19T16:14:44,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9e4d51e1b8cb454a97e49236296d3265 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9e4d51e1b8cb454a97e49236296d3265 2024-11-19T16:14:44,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9984443b7f6440e399e7954614ce925d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/9984443b7f6440e399e7954614ce925d 2024-11-19T16:14:44,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/bf7567d6eb154287bb03ef424305a482 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/bf7567d6eb154287bb03ef424305a482 2024-11-19T16:14:44,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/61b9956f9f004a75aef4274ac4484ee6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/61b9956f9f004a75aef4274ac4484ee6 2024-11-19T16:14:44,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/5bf8781cf1c94392a6236a768fee31b8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/5bf8781cf1c94392a6236a768fee31b8 2024-11-19T16:14:44,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/f10b00c76f3047dfa7bfe2601e5d4aeb to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/f10b00c76f3047dfa7bfe2601e5d4aeb 2024-11-19T16:14:44,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/3b2340a630d948478cbd67f907d6e7ab to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/3b2340a630d948478cbd67f907d6e7ab 2024-11-19T16:14:44,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/4ee0e9ae124a4c1c80e3d62dae70294e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/4ee0e9ae124a4c1c80e3d62dae70294e 2024-11-19T16:14:44,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/48117146fd9548d689d01144b07d32d4 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/48117146fd9548d689d01144b07d32d4 2024-11-19T16:14:44,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/0aca97709d1b4f89b72cf5c05bac735b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/0aca97709d1b4f89b72cf5c05bac735b 2024-11-19T16:14:44,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/00d72f4408034854a6d8f4c3fcf014cc to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/00d72f4408034854a6d8f4c3fcf014cc 2024-11-19T16:14:44,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/417cce7605894c77b399c176ae950da9 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/417cce7605894c77b399c176ae950da9 2024-11-19T16:14:44,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/1e3e0c63f2334da695ad8315e603c65d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/1e3e0c63f2334da695ad8315e603c65d 2024-11-19T16:14:44,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e59caa670b184f37af6f67dbe117803b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/e59caa670b184f37af6f67dbe117803b 2024-11-19T16:14:44,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/c681496ea1aa4e4db677acc9a65b8378 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/c681496ea1aa4e4db677acc9a65b8378 2024-11-19T16:14:44,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/fa220d6869b949409735d81c11460b4a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/fa220d6869b949409735d81c11460b4a 2024-11-19T16:14:44,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/2fb4306113c047588c67d631ff8b1323 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/2fb4306113c047588c67d631ff8b1323 2024-11-19T16:14:44,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/7a9468b51a14474485aa46e9f58828b7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/7a9468b51a14474485aa46e9f58828b7 2024-11-19T16:14:44,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/61971514c7744f369efebc7b2eae246c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/61971514c7744f369efebc7b2eae246c 2024-11-19T16:14:44,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/964cf1baa5bd449fbb9bc19aec324581 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/964cf1baa5bd449fbb9bc19aec324581 2024-11-19T16:14:44,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/7ce558c79a1040778685aec8271100b3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d45d947746de4a70b848ff4a6d9697fd, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ce03744cc750413aaeb564b7ea2b4138, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b8504b9010134e0aab296d8d3663f108, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/f1d36ac3ef024b729aab7df0efffdb05, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/81bca55a5afb48988baaff45ef04eb3d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b78dbec3945e4b149ed814c1c127aca7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/202d06a034774a40a1ab64b8cb0b0a13, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/327e091321a342aca9375f82b037196b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/492b94e1e4e346c6baaf2ba796bc5f51, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d0abc96a0c6942349dcf78e4bd52dce1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/e068eb7f61f64930b1c4b1e1131fca74, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d3a2c791aed448538d5cd98093a73607, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8b4e72517aff47fe89ab9d2cd5fd4491, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/f6d9777803d2445da5ccff026f5dada0, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b99b2b00b0134c7a973a20d971d213dc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/056c5f7154ab42f095a4545c2c02fd81, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/9b9860d023514186bbfde9af44a400e8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/3ff9090a6b27427e96bf14d259ad1143, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b4fe97e0606e4956a862b45de6b68656, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8179c7baed5e417a8e307d03ee30a8b7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/fedff7b5b0f74106b354e56409c930d1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ee38b75fbc854c0ba394306eacfaf5fc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/36114f21d6714216a1ea073f01840ff4, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/08a3673a52ad4056a1861b72936f5176, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/15eacfe1aae040d3979528032a7a2c40, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/50d0fc03941a4dccb5d2fead33fcb0ff, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/7de18bfbd7984ba2995a19689ab00bdf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ea6a515d56ce44158171e7ee9c610279, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8fb4992edf614b239cd506a743513e02, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/67473bfcf57f44d282e7833af94cf253, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/6d4a787b4cff4fa6838d3e8768cadc6e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/73298119610e445b95f823953ec5a651, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/316cfdaa982347148e98b0f28e24e953] to archive 2024-11-19T16:14:44,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:14:44,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/7ce558c79a1040778685aec8271100b3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/7ce558c79a1040778685aec8271100b3 2024-11-19T16:14:44,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d45d947746de4a70b848ff4a6d9697fd to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d45d947746de4a70b848ff4a6d9697fd 2024-11-19T16:14:44,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ce03744cc750413aaeb564b7ea2b4138 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ce03744cc750413aaeb564b7ea2b4138 2024-11-19T16:14:44,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b8504b9010134e0aab296d8d3663f108 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b8504b9010134e0aab296d8d3663f108 2024-11-19T16:14:44,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/f1d36ac3ef024b729aab7df0efffdb05 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/f1d36ac3ef024b729aab7df0efffdb05 2024-11-19T16:14:44,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/81bca55a5afb48988baaff45ef04eb3d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/81bca55a5afb48988baaff45ef04eb3d 2024-11-19T16:14:44,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b78dbec3945e4b149ed814c1c127aca7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b78dbec3945e4b149ed814c1c127aca7 2024-11-19T16:14:44,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/202d06a034774a40a1ab64b8cb0b0a13 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/202d06a034774a40a1ab64b8cb0b0a13 2024-11-19T16:14:44,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/327e091321a342aca9375f82b037196b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/327e091321a342aca9375f82b037196b 2024-11-19T16:14:44,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/492b94e1e4e346c6baaf2ba796bc5f51 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/492b94e1e4e346c6baaf2ba796bc5f51 2024-11-19T16:14:44,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d0abc96a0c6942349dcf78e4bd52dce1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d0abc96a0c6942349dcf78e4bd52dce1 2024-11-19T16:14:44,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/e068eb7f61f64930b1c4b1e1131fca74 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/e068eb7f61f64930b1c4b1e1131fca74 2024-11-19T16:14:44,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d3a2c791aed448538d5cd98093a73607 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d3a2c791aed448538d5cd98093a73607 2024-11-19T16:14:44,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8b4e72517aff47fe89ab9d2cd5fd4491 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8b4e72517aff47fe89ab9d2cd5fd4491 2024-11-19T16:14:44,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/f6d9777803d2445da5ccff026f5dada0 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/f6d9777803d2445da5ccff026f5dada0 2024-11-19T16:14:44,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b99b2b00b0134c7a973a20d971d213dc to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b99b2b00b0134c7a973a20d971d213dc 2024-11-19T16:14:44,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/056c5f7154ab42f095a4545c2c02fd81 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/056c5f7154ab42f095a4545c2c02fd81 2024-11-19T16:14:44,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/9b9860d023514186bbfde9af44a400e8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/9b9860d023514186bbfde9af44a400e8 2024-11-19T16:14:44,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/3ff9090a6b27427e96bf14d259ad1143 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/3ff9090a6b27427e96bf14d259ad1143 2024-11-19T16:14:44,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b4fe97e0606e4956a862b45de6b68656 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/b4fe97e0606e4956a862b45de6b68656 2024-11-19T16:14:44,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8179c7baed5e417a8e307d03ee30a8b7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8179c7baed5e417a8e307d03ee30a8b7 2024-11-19T16:14:44,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/fedff7b5b0f74106b354e56409c930d1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/fedff7b5b0f74106b354e56409c930d1 2024-11-19T16:14:44,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ee38b75fbc854c0ba394306eacfaf5fc to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ee38b75fbc854c0ba394306eacfaf5fc 2024-11-19T16:14:44,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/36114f21d6714216a1ea073f01840ff4 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/36114f21d6714216a1ea073f01840ff4 2024-11-19T16:14:44,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/08a3673a52ad4056a1861b72936f5176 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/08a3673a52ad4056a1861b72936f5176 2024-11-19T16:14:44,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/15eacfe1aae040d3979528032a7a2c40 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/15eacfe1aae040d3979528032a7a2c40 2024-11-19T16:14:44,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/50d0fc03941a4dccb5d2fead33fcb0ff to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/50d0fc03941a4dccb5d2fead33fcb0ff 2024-11-19T16:14:44,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/7de18bfbd7984ba2995a19689ab00bdf to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/7de18bfbd7984ba2995a19689ab00bdf 2024-11-19T16:14:44,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ea6a515d56ce44158171e7ee9c610279 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/ea6a515d56ce44158171e7ee9c610279 2024-11-19T16:14:44,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8fb4992edf614b239cd506a743513e02 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/8fb4992edf614b239cd506a743513e02 2024-11-19T16:14:44,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/67473bfcf57f44d282e7833af94cf253 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/67473bfcf57f44d282e7833af94cf253 2024-11-19T16:14:44,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/6d4a787b4cff4fa6838d3e8768cadc6e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/6d4a787b4cff4fa6838d3e8768cadc6e 2024-11-19T16:14:44,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/73298119610e445b95f823953ec5a651 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/73298119610e445b95f823953ec5a651 2024-11-19T16:14:44,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/316cfdaa982347148e98b0f28e24e953 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/316cfdaa982347148e98b0f28e24e953 2024-11-19T16:14:44,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/bbe3eae4be2c489cb28d7d2d27a04a70, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/b51de2ddff694127bd9b3570b11f25bd, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/e65e2ed2e89c42588b9e50d04d616a09, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/75734e56028b4c44aaa634f4477a8978, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/4b4e5bc1f2584a329e288ae160b9f9e6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/39cc7ac25e5146e6a610d017fa9f5f65, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/632f2e9ab9d24c71b5f8118129aadd53, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/90142d7e5d5542ba8cc2e8a65e2fe8b3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/9bc15e56f6db4d5796b77a2b7592b715, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/015d60bcb91541f593222e65e317a920, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1064ea77dbc94cfc980fcddb8fdcab79, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/06c6134acb814a9faa015ad52a1d6338, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/c12154bb191b48dd96edf85efa816c9b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/9e5fb03da8d74f23a277b958d6782d4f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/726c232a2d8845eab064357529e9e9a0, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1a7ca5b1c9c1446fad61f9fc38c81b45, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1260b6087cff494fbb75142ca51af827, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/655a70e7649a466d98f8514f7597dafa, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/7c6c2b9d7f2c490fbbb8c31f46791682, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/ab0793503d2f45dd872911cae83e0595, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/3f0d3e3abc384366aaf37b3b21ed3938, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a02a81bfb70642e09cc3c5eac4ebd349, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/bbc35bc68bd548f991c7a3b8078b168e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/23da7105cf5b4a9ba4f0d468887c20bb, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/0dcfb3ef93a6470d86ce8c8d760689b8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/b09ef57c94cb4a2281c505f265e7e807, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/0436a870b6b246d18e70c6a32789a122, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/94f4226cdf6441b7acc2084fd4002e68, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/4ca1876022744d22866cca93fd0c3b9d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/fded48ed1b8b400fa52ab4eec895ac40, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a15ce2e684e34f958380268328cfdab7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/c3dbc314d7a146378bdf8b249e68125b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/2475176dddef4ab4b7361c916c3553eb, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/46ff5b676e4846c9bfb07d6ef8723cf6] to archive 2024-11-19T16:14:44,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:14:44,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/bbe3eae4be2c489cb28d7d2d27a04a70 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/bbe3eae4be2c489cb28d7d2d27a04a70 2024-11-19T16:14:44,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/b51de2ddff694127bd9b3570b11f25bd to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/b51de2ddff694127bd9b3570b11f25bd 2024-11-19T16:14:44,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/e65e2ed2e89c42588b9e50d04d616a09 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/e65e2ed2e89c42588b9e50d04d616a09 2024-11-19T16:14:44,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/75734e56028b4c44aaa634f4477a8978 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/75734e56028b4c44aaa634f4477a8978 2024-11-19T16:14:44,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/4b4e5bc1f2584a329e288ae160b9f9e6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/4b4e5bc1f2584a329e288ae160b9f9e6 2024-11-19T16:14:44,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/39cc7ac25e5146e6a610d017fa9f5f65 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/39cc7ac25e5146e6a610d017fa9f5f65 2024-11-19T16:14:44,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/632f2e9ab9d24c71b5f8118129aadd53 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/632f2e9ab9d24c71b5f8118129aadd53 2024-11-19T16:14:44,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/90142d7e5d5542ba8cc2e8a65e2fe8b3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/90142d7e5d5542ba8cc2e8a65e2fe8b3 2024-11-19T16:14:44,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/9bc15e56f6db4d5796b77a2b7592b715 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/9bc15e56f6db4d5796b77a2b7592b715 2024-11-19T16:14:44,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/015d60bcb91541f593222e65e317a920 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/015d60bcb91541f593222e65e317a920 2024-11-19T16:14:44,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1064ea77dbc94cfc980fcddb8fdcab79 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1064ea77dbc94cfc980fcddb8fdcab79 2024-11-19T16:14:44,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/06c6134acb814a9faa015ad52a1d6338 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/06c6134acb814a9faa015ad52a1d6338 2024-11-19T16:14:44,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/c12154bb191b48dd96edf85efa816c9b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/c12154bb191b48dd96edf85efa816c9b 2024-11-19T16:14:44,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/9e5fb03da8d74f23a277b958d6782d4f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/9e5fb03da8d74f23a277b958d6782d4f 2024-11-19T16:14:44,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/726c232a2d8845eab064357529e9e9a0 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/726c232a2d8845eab064357529e9e9a0 2024-11-19T16:14:44,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1a7ca5b1c9c1446fad61f9fc38c81b45 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1a7ca5b1c9c1446fad61f9fc38c81b45 2024-11-19T16:14:44,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1260b6087cff494fbb75142ca51af827 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/1260b6087cff494fbb75142ca51af827 2024-11-19T16:14:44,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/655a70e7649a466d98f8514f7597dafa to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/655a70e7649a466d98f8514f7597dafa 2024-11-19T16:14:44,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/7c6c2b9d7f2c490fbbb8c31f46791682 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/7c6c2b9d7f2c490fbbb8c31f46791682 2024-11-19T16:14:44,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/ab0793503d2f45dd872911cae83e0595 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/ab0793503d2f45dd872911cae83e0595 2024-11-19T16:14:44,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/3f0d3e3abc384366aaf37b3b21ed3938 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/3f0d3e3abc384366aaf37b3b21ed3938 2024-11-19T16:14:44,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a02a81bfb70642e09cc3c5eac4ebd349 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a02a81bfb70642e09cc3c5eac4ebd349 2024-11-19T16:14:44,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/bbc35bc68bd548f991c7a3b8078b168e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/bbc35bc68bd548f991c7a3b8078b168e 2024-11-19T16:14:44,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/23da7105cf5b4a9ba4f0d468887c20bb to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/23da7105cf5b4a9ba4f0d468887c20bb 2024-11-19T16:14:44,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/0dcfb3ef93a6470d86ce8c8d760689b8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/0dcfb3ef93a6470d86ce8c8d760689b8 2024-11-19T16:14:44,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/b09ef57c94cb4a2281c505f265e7e807 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/b09ef57c94cb4a2281c505f265e7e807 2024-11-19T16:14:44,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/0436a870b6b246d18e70c6a32789a122 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/0436a870b6b246d18e70c6a32789a122 2024-11-19T16:14:44,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/94f4226cdf6441b7acc2084fd4002e68 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/94f4226cdf6441b7acc2084fd4002e68 2024-11-19T16:14:44,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/4ca1876022744d22866cca93fd0c3b9d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/4ca1876022744d22866cca93fd0c3b9d 2024-11-19T16:14:44,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/fded48ed1b8b400fa52ab4eec895ac40 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/fded48ed1b8b400fa52ab4eec895ac40 2024-11-19T16:14:44,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a15ce2e684e34f958380268328cfdab7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a15ce2e684e34f958380268328cfdab7 2024-11-19T16:14:44,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/c3dbc314d7a146378bdf8b249e68125b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/c3dbc314d7a146378bdf8b249e68125b 2024-11-19T16:14:44,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/2475176dddef4ab4b7361c916c3553eb to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/2475176dddef4ab4b7361c916c3553eb 2024-11-19T16:14:44,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/46ff5b676e4846c9bfb07d6ef8723cf6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/46ff5b676e4846c9bfb07d6ef8723cf6 2024-11-19T16:14:44,792 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/recovered.edits/715.seqid, newMaxSeqId=715, maxSeqId=1 2024-11-19T16:14:44,793 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. 2024-11-19T16:14:44,793 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for c20cca7a0cf0913bda0c7ed239b24e88: Waiting for close lock at 1732032882865Running coprocessor pre-close hooks at 1732032882865Disabling compacts and flushes for region at 1732032882865Disabling writes for close at 1732032883322 (+457 ms)Obtaining lock to block concurrent updates at 1732032883322Preparing flush snapshotting stores in c20cca7a0cf0913bda0c7ed239b24e88 at 1732032883322Finished memstore snapshotting TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732032883323 (+1 ms)Flushing stores of TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88. at 1732032883323Flushing c20cca7a0cf0913bda0c7ed239b24e88/A: creating writer at 1732032883323Flushing c20cca7a0cf0913bda0c7ed239b24e88/A: appending metadata at 1732032883326 (+3 ms)Flushing c20cca7a0cf0913bda0c7ed239b24e88/A: closing flushed file at 1732032883326Flushing c20cca7a0cf0913bda0c7ed239b24e88/B: creating writer at 1732032883737 (+411 ms)Flushing c20cca7a0cf0913bda0c7ed239b24e88/B: appending metadata at 1732032883740 (+3 ms)Flushing c20cca7a0cf0913bda0c7ed239b24e88/B: closing flushed file at 1732032883741 (+1 ms)Flushing c20cca7a0cf0913bda0c7ed239b24e88/C: creating writer at 1732032884204 (+463 ms)Flushing c20cca7a0cf0913bda0c7ed239b24e88/C: appending metadata at 1732032884206 (+2 ms)Flushing c20cca7a0cf0913bda0c7ed239b24e88/C: closing flushed file at 1732032884206Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c6ef0f2: reopening flushed file at 1732032884616 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4975c615: reopening flushed file at 1732032884621 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46c9a216: reopening flushed file at 1732032884627 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c20cca7a0cf0913bda0c7ed239b24e88 in 1310ms, sequenceid=712, compaction requested=true at 1732032884633 (+6 ms)Writing region close event to WAL at 1732032884788 (+155 ms)Running coprocessor post-close hooks at 1732032884793 (+5 ms)Closed at 1732032884793 2024-11-19T16:14:44,796 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:44,796 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=c20cca7a0cf0913bda0c7ed239b24e88, regionState=CLOSED 2024-11-19T16:14:44,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:14:44,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-19T16:14:44,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure c20cca7a0cf0913bda0c7ed239b24e88, server=32a8860bb9d6,43095,1732032854050 in 2.0950 sec 2024-11-19T16:14:44,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-19T16:14:44,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c20cca7a0cf0913bda0c7ed239b24e88, UNASSIGN in 2.1020 sec 2024-11-19T16:14:44,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-19T16:14:44,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1080 sec 2024-11-19T16:14:44,808 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032884807"}]},"ts":"1732032884807"} 2024-11-19T16:14:44,809 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T16:14:44,809 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T16:14:44,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1230 sec 2024-11-19T16:14:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T16:14:44,837 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:44,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-19T16:14:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:14:44,845 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:14:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T16:14:44,847 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:14:44,853 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:44,858 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/recovered.edits] 2024-11-19T16:14:44,861 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/44e2f260e9bc49e9af459079a5f50c10 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/44e2f260e9bc49e9af459079a5f50c10 2024-11-19T16:14:44,863 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/7d8fa8cb20c24c5eaeaac1fbbb159417 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/7d8fa8cb20c24c5eaeaac1fbbb159417 2024-11-19T16:14:44,864 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/996d127c1a0a4aaca1aa989b353326fc to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/996d127c1a0a4aaca1aa989b353326fc 2024-11-19T16:14:44,866 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/b9e6aebbfdea40f8afbf7dc18c8d91ca to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/A/b9e6aebbfdea40f8afbf7dc18c8d91ca 2024-11-19T16:14:44,868 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/c71c3c20a062460088fd6c1c3442a67f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/c71c3c20a062460088fd6c1c3442a67f 2024-11-19T16:14:44,870 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d489c0344674473e8a95250b70fbc672 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/d489c0344674473e8a95250b70fbc672 2024-11-19T16:14:44,871 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/e262596e3a1c4e61939ca7d0721795ff to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/e262596e3a1c4e61939ca7d0721795ff 2024-11-19T16:14:44,872 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/fc819121292b4aae8299208181fdca79 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/B/fc819121292b4aae8299208181fdca79 2024-11-19T16:14:44,875 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/2225317569d3455d8ae6ccf40b13c483 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/2225317569d3455d8ae6ccf40b13c483 2024-11-19T16:14:44,876 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/3db452fedf554a5cb5fc05d28c0e8da6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/3db452fedf554a5cb5fc05d28c0e8da6 2024-11-19T16:14:44,877 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/966724dbaec940cfb496c92ea25afbb4 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/966724dbaec940cfb496c92ea25afbb4 2024-11-19T16:14:44,879 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a8e6fc006a6d4582b173b25793e24bc2 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/C/a8e6fc006a6d4582b173b25793e24bc2 2024-11-19T16:14:44,881 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/recovered.edits/715.seqid to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88/recovered.edits/715.seqid 2024-11-19T16:14:44,882 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/c20cca7a0cf0913bda0c7ed239b24e88 2024-11-19T16:14:44,882 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T16:14:44,887 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:14:44,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-19T16:14:44,896 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T16:14:44,927 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T16:14:44,929 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:14:44,929 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T16:14:44,929 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732032884929"}]},"ts":"9223372036854775807"} 2024-11-19T16:14:44,933 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T16:14:44,933 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c20cca7a0cf0913bda0c7ed239b24e88, NAME => 'TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T16:14:44,934 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T16:14:44,934 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732032884934"}]},"ts":"9223372036854775807"} 2024-11-19T16:14:44,937 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T16:14:44,938 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:14:44,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 97 msec 2024-11-19T16:14:44,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T16:14:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T16:14:44,958 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:44,967 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-607038221_22 at /127.0.0.1:60138 [Waiting for operation #355] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2117921560_22 at /127.0.0.1:54708 [Waiting for operation #346] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;32a8860bb9d6:43095-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=377 (was 304) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4385 (was 4953) 2024-11-19T16:14:44,976 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=377, ProcessCount=11, AvailableMemoryMB=4383 2024-11-19T16:14:44,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T16:14:44,978 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T16:14:44,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T16:14:44,980 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T16:14:44,980 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:44,980 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-19T16:14:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-19T16:14:44,981 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T16:14:44,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741951_1127 (size=1061) 2024-11-19T16:14:45,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-19T16:14:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-19T16:14:45,394 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70 2024-11-19T16:14:45,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741952_1128 (size=53) 2024-11-19T16:14:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-19T16:14:45,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:14:45,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 828823ece6f79c9635b39bcf387a8d6c, disabling compactions & flushes 2024-11-19T16:14:45,804 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:45,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:45,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. after waiting 0 ms 2024-11-19T16:14:45,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:45,804 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:45,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 828823ece6f79c9635b39bcf387a8d6c: Waiting for close lock at 1732032885804Disabling compacts and flushes for region at 1732032885804Disabling writes for close at 1732032885804Writing region close event to WAL at 1732032885804Closed at 1732032885804 2024-11-19T16:14:45,806 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T16:14:45,806 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732032885806"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732032885806"}]},"ts":"1732032885806"} 2024-11-19T16:14:45,808 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T16:14:45,809 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T16:14:45,809 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032885809"}]},"ts":"1732032885809"} 2024-11-19T16:14:45,810 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T16:14:45,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, ASSIGN}] 2024-11-19T16:14:45,812 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, ASSIGN 2024-11-19T16:14:45,813 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, ASSIGN; state=OFFLINE, location=32a8860bb9d6,43095,1732032854050; forceNewPlan=false, retain=false 2024-11-19T16:14:45,964 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=828823ece6f79c9635b39bcf387a8d6c, regionState=OPENING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:45,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, ASSIGN because future has completed 2024-11-19T16:14:45,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:14:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-19T16:14:46,128 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:46,129 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} 2024-11-19T16:14:46,129 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:46,129 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:14:46,129 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:46,129 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:46,131 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:46,132 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:14:46,133 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 828823ece6f79c9635b39bcf387a8d6c columnFamilyName A 2024-11-19T16:14:46,133 DEBUG [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:46,134 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.HStore(327): Store=828823ece6f79c9635b39bcf387a8d6c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:14:46,134 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:46,135 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:14:46,135 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 828823ece6f79c9635b39bcf387a8d6c columnFamilyName B 2024-11-19T16:14:46,135 DEBUG [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:46,136 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.HStore(327): Store=828823ece6f79c9635b39bcf387a8d6c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:14:46,136 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:46,137 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:14:46,137 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 828823ece6f79c9635b39bcf387a8d6c columnFamilyName C 2024-11-19T16:14:46,138 DEBUG [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:46,138 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.HStore(327): Store=828823ece6f79c9635b39bcf387a8d6c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:14:46,138 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:46,138 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:46,139 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:46,140 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:46,141 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:46,141 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:46,141 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T16:14:46,142 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:46,144 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T16:14:46,144 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 828823ece6f79c9635b39bcf387a8d6c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67861602, jitterRate=0.011216670274734497}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T16:14:46,145 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:46,145 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 828823ece6f79c9635b39bcf387a8d6c: Running coprocessor pre-open hook at 1732032886129Writing region info on filesystem at 1732032886129Initializing all the Stores at 1732032886130 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032886130Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032886130Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032886130Cleaning up temporary data from old regions at 1732032886141 (+11 ms)Running coprocessor post-open hooks at 1732032886145 (+4 ms)Region opened successfully at 1732032886145 2024-11-19T16:14:46,146 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., pid=32, masterSystemTime=1732032886125 2024-11-19T16:14:46,148 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:46,148 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:46,149 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=828823ece6f79c9635b39bcf387a8d6c, regionState=OPEN, openSeqNum=2, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:46,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:14:46,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-19T16:14:46,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 in 182 msec 2024-11-19T16:14:46,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-19T16:14:46,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, ASSIGN in 343 msec 2024-11-19T16:14:46,157 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T16:14:46,157 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032886157"}]},"ts":"1732032886157"} 2024-11-19T16:14:46,159 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T16:14:46,160 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T16:14:46,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-19T16:14:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-19T16:14:47,128 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:47,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:47,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:47,132 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23bc00e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:47,136 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93c960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,141 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:47,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:47,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:47,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:47,147 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T16:14:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 32a8860bb9d6,37355,1732032853237 2024-11-19T16:14:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2022ca19 2024-11-19T16:14:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T16:14:47,151 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T16:14:47,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T16:14:47,159 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T16:14:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T16:14:47,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741953_1129 (size=1097) 2024-11-19T16:14:47,194 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T16:14:47,194 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T16:14:47,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T16:14:47,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, REOPEN/MOVE}] 2024-11-19T16:14:47,208 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, REOPEN/MOVE 2024-11-19T16:14:47,208 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=828823ece6f79c9635b39bcf387a8d6c, regionState=CLOSING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:47,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, REOPEN/MOVE because future has completed 2024-11-19T16:14:47,211 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T16:14:47,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:14:47,364 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,364 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T16:14:47,364 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 828823ece6f79c9635b39bcf387a8d6c, disabling compactions & flushes 2024-11-19T16:14:47,364 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:47,364 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:47,364 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. after waiting 0 ms 2024-11-19T16:14:47,364 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:47,371 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T16:14:47,372 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:47,372 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 828823ece6f79c9635b39bcf387a8d6c: Waiting for close lock at 1732032887364Running coprocessor pre-close hooks at 1732032887364Disabling compacts and flushes for region at 1732032887364Disabling writes for close at 1732032887364Writing region close event to WAL at 1732032887365 (+1 ms)Running coprocessor post-close hooks at 1732032887372 (+7 ms)Closed at 1732032887372 2024-11-19T16:14:47,373 WARN [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 828823ece6f79c9635b39bcf387a8d6c to self. 2024-11-19T16:14:47,375 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,378 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=828823ece6f79c9635b39bcf387a8d6c, regionState=CLOSED 2024-11-19T16:14:47,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:14:47,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-19T16:14:47,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 in 171 msec 2024-11-19T16:14:47,385 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, REOPEN/MOVE; state=CLOSED, location=32a8860bb9d6,43095,1732032854050; forceNewPlan=false, retain=true 2024-11-19T16:14:47,537 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=828823ece6f79c9635b39bcf387a8d6c, regionState=OPENING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:47,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, REOPEN/MOVE because future has completed 2024-11-19T16:14:47,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:14:47,703 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:47,703 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} 2024-11-19T16:14:47,704 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,704 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:14:47,704 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,704 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,709 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,709 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:14:47,713 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 828823ece6f79c9635b39bcf387a8d6c columnFamilyName A 2024-11-19T16:14:47,715 DEBUG [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:47,716 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.HStore(327): Store=828823ece6f79c9635b39bcf387a8d6c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:14:47,716 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,717 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:14:47,717 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 828823ece6f79c9635b39bcf387a8d6c columnFamilyName B 2024-11-19T16:14:47,717 DEBUG [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:47,717 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.HStore(327): Store=828823ece6f79c9635b39bcf387a8d6c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:14:47,717 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,718 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:14:47,718 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 828823ece6f79c9635b39bcf387a8d6c columnFamilyName C 2024-11-19T16:14:47,718 DEBUG [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:47,719 INFO [StoreOpener-828823ece6f79c9635b39bcf387a8d6c-1 {}] regionserver.HStore(327): Store=828823ece6f79c9635b39bcf387a8d6c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:14:47,719 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:47,719 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,719 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,720 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,721 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,721 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,722 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T16:14:47,723 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,723 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 828823ece6f79c9635b39bcf387a8d6c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65985604, jitterRate=-0.01673787832260132}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T16:14:47,723 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,724 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 828823ece6f79c9635b39bcf387a8d6c: Running coprocessor pre-open hook at 1732032887704Writing region info on filesystem at 1732032887704Initializing all the Stores at 1732032887705 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032887705Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032887705Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032887705Cleaning up temporary data from old regions at 1732032887721 (+16 ms)Running coprocessor post-open hooks at 1732032887723 (+2 ms)Region opened successfully at 1732032887724 (+1 ms) 2024-11-19T16:14:47,725 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., pid=37, masterSystemTime=1732032887696 2024-11-19T16:14:47,726 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:47,726 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:47,727 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=828823ece6f79c9635b39bcf387a8d6c, regionState=OPEN, openSeqNum=5, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:47,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:14:47,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-19T16:14:47,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 in 188 msec 2024-11-19T16:14:47,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-19T16:14:47,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, REOPEN/MOVE in 524 msec 2024-11-19T16:14:47,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-19T16:14:47,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 535 msec 2024-11-19T16:14:47,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 574 msec 2024-11-19T16:14:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-19T16:14:47,741 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:47,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,742 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:47,742 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:47,743 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78ba1417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:47,744 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1913bfac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:47,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:47,747 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6814e038, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:47,749 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5faef52d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:47,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:47,752 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad2eec7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:47,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:47,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:47,754 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@335352bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:47,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:47,756 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50e3ba55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:47,758 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3f8a8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:47,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:47,760 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f938c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:47,762 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@609761c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:47,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:47,765 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@255117c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:47,767 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:47,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53b6ebb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:47,771 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:47,772 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:47,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:47,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f235704, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:47,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:47,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:47,774 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763f036f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,775 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:47,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:47,776 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:47,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:47,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:47,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dcc0eb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:47,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:47,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:47,778 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d31dbd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:14:47,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:14:47,780 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:14:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:14:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:14:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4afa3bf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:14:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:14:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:14:47,782 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:14:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594dbf65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:14:47,783 DEBUG [Thread-655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:47,783 DEBUG [Thread-657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:47,783 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:47,783 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:47,783 DEBUG [Thread-661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:47,784 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:47,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:47,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:47,785 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:47,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:47,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:47,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:47,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:47,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:47,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:47,786 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:47,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:47,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-19T16:14:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:47,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:47,789 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:47,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:47,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:47,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:47,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:47,790 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T16:14:47,791 DEBUG [Thread-644 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:14:47,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:14:47,792 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:47,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:14:47,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:14:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:14:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:14:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:14:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:14:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:14:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:14:47,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:14:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:14:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:14:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:14:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:14:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:14:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:14:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:47,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032887861, max timestamp=1732032887878Num uniques 859; 2024-11-19T16:14:47,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:14:47,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032887861, max timestamp=1732032887878Num uniques 859; 2024-11-19T16:14:47,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:14:47,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032887861, max timestamp=1732032887878Num uniques 859; 2024-11-19T16:14:47,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:14:47,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193733b8c82fc9426bb86904d444026260_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032887819/Put/seqid=0 2024-11-19T16:14:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T16:14:47,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741954_1130 (size=12154) 2024-11-19T16:14:47,904 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:47,911 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193733b8c82fc9426bb86904d444026260_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193733b8c82fc9426bb86904d444026260_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:47,913 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/42b75fa0ecae4a3f8c633d428fde5e6e, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:47,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032947916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:47,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032947919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:47,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/42b75fa0ecae4a3f8c633d428fde5e6e is 175, key is test_row_0/A:col10/1732032887819/Put/seqid=0 2024-11-19T16:14:47,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032947924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:47,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032947924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:47,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032947926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:47,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T16:14:47,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:47,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:47,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:47,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:47,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:47,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741955_1131 (size=30955) 2024-11-19T16:14:47,979 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/42b75fa0ecae4a3f8c633d428fde5e6e 2024-11-19T16:14:48,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/2b060cbea3f141ffb52499544d6ca75b is 50, key is test_row_0/B:col10/1732032887819/Put/seqid=0 2024-11-19T16:14:48,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741956_1132 (size=12001) 2024-11-19T16:14:48,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/2b060cbea3f141ffb52499544d6ca75b 2024-11-19T16:14:48,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032948037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032948037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032948038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032948038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032948047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/af7584d3d9b94ceebfac478b8d819116 is 50, key is test_row_0/C:col10/1732032887819/Put/seqid=0 2024-11-19T16:14:48,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T16:14:48,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:48,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:48,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:48,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:48,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:48,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741957_1133 (size=12001) 2024-11-19T16:14:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T16:14:48,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/af7584d3d9b94ceebfac478b8d819116 2024-11-19T16:14:48,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/42b75fa0ecae4a3f8c633d428fde5e6e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42b75fa0ecae4a3f8c633d428fde5e6e 2024-11-19T16:14:48,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42b75fa0ecae4a3f8c633d428fde5e6e, entries=150, sequenceid=14, filesize=30.2 K 2024-11-19T16:14:48,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/2b060cbea3f141ffb52499544d6ca75b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/2b060cbea3f141ffb52499544d6ca75b 2024-11-19T16:14:48,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/2b060cbea3f141ffb52499544d6ca75b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T16:14:48,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/af7584d3d9b94ceebfac478b8d819116 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/af7584d3d9b94ceebfac478b8d819116 2024-11-19T16:14:48,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/af7584d3d9b94ceebfac478b8d819116, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T16:14:48,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 828823ece6f79c9635b39bcf387a8d6c in 319ms, sequenceid=14, compaction requested=false 2024-11-19T16:14:48,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:48,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T16:14:48,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:48,256 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-19T16:14:48,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:14:48,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:48,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:14:48,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:48,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:14:48,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:48,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:48,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119493f1bbcc3364d33b1f5525dbee316b9_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032887923/Put/seqid=0 2024-11-19T16:14:48,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032948271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032948270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032948272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032948286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741958_1134 (size=12154) 2024-11-19T16:14:48,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:48,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032948291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,302 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119493f1bbcc3364d33b1f5525dbee316b9_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119493f1bbcc3364d33b1f5525dbee316b9_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:48,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/06cf6b9f5aac43e09e7154b62bad6da9, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:48,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/06cf6b9f5aac43e09e7154b62bad6da9 is 175, key is test_row_0/A:col10/1732032887923/Put/seqid=0 2024-11-19T16:14:48,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741959_1135 (size=30955) 2024-11-19T16:14:48,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032948396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032948397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032948397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032948397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032948406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T16:14:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032948607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032948608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032948608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032948609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032948617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,718 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/06cf6b9f5aac43e09e7154b62bad6da9 2024-11-19T16:14:48,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/65172e8e819e47c1b2d612e413f66ea3 is 50, key is test_row_0/B:col10/1732032887923/Put/seqid=0 2024-11-19T16:14:48,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741960_1136 (size=12001) 2024-11-19T16:14:48,738 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/65172e8e819e47c1b2d612e413f66ea3 2024-11-19T16:14:48,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/46551c4f955749249a791aca1792a12e is 50, key is test_row_0/C:col10/1732032887923/Put/seqid=0 2024-11-19T16:14:48,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741961_1137 (size=12001) 2024-11-19T16:14:48,764 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/46551c4f955749249a791aca1792a12e 2024-11-19T16:14:48,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/06cf6b9f5aac43e09e7154b62bad6da9 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/06cf6b9f5aac43e09e7154b62bad6da9 2024-11-19T16:14:48,779 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/06cf6b9f5aac43e09e7154b62bad6da9, entries=150, sequenceid=49, filesize=30.2 K 2024-11-19T16:14:48,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/65172e8e819e47c1b2d612e413f66ea3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/65172e8e819e47c1b2d612e413f66ea3 2024-11-19T16:14:48,785 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/65172e8e819e47c1b2d612e413f66ea3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T16:14:48,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/46551c4f955749249a791aca1792a12e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/46551c4f955749249a791aca1792a12e 2024-11-19T16:14:48,792 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/46551c4f955749249a791aca1792a12e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T16:14:48,793 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 828823ece6f79c9635b39bcf387a8d6c in 537ms, sequenceid=49, compaction requested=false 2024-11-19T16:14:48,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:48,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:48,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-19T16:14:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-19T16:14:48,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-19T16:14:48,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0020 sec 2024-11-19T16:14:48,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.0130 sec 2024-11-19T16:14:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:48,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T16:14:48,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:14:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:14:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:14:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T16:14:48,927 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:48,928 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-19T16:14:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T16:14:48,931 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:48,931 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:48,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:48,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198d2a4291ff6649a89a3d33cf9e7d390b_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032888283/Put/seqid=0 2024-11-19T16:14:48,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032948950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032948951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032948953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032948957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032948959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:48,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741962_1138 (size=12154) 2024-11-19T16:14:48,971 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:48,979 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198d2a4291ff6649a89a3d33cf9e7d390b_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198d2a4291ff6649a89a3d33cf9e7d390b_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:48,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/6a8a538b58c5403b8cb21c29cfa778d3, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:48,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/6a8a538b58c5403b8cb21c29cfa778d3 is 175, key is test_row_0/A:col10/1732032888283/Put/seqid=0 2024-11-19T16:14:48,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741963_1139 (size=30955) 2024-11-19T16:14:48,999 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/6a8a538b58c5403b8cb21c29cfa778d3 2024-11-19T16:14:49,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/511c3ae536914114b3917c0ee8ad031a is 50, key is test_row_0/B:col10/1732032888283/Put/seqid=0 2024-11-19T16:14:49,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741964_1140 (size=12001) 2024-11-19T16:14:49,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/511c3ae536914114b3917c0ee8ad031a 2024-11-19T16:14:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T16:14:49,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/0b701aec8c7f4cb7a4e20908f872361f is 50, key is test_row_0/C:col10/1732032888283/Put/seqid=0 2024-11-19T16:14:49,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032949066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741965_1141 (size=12001) 2024-11-19T16:14:49,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032949067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032949067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032949068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032949068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T16:14:49,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:49,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:49,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:49,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:49,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:49,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T16:14:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:49,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:49,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T16:14:49,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032949277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032949277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032949277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032949278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032949278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T16:14:49,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:49,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:49,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:49,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:49,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:49,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:49,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/0b701aec8c7f4cb7a4e20908f872361f 2024-11-19T16:14:49,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/6a8a538b58c5403b8cb21c29cfa778d3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/6a8a538b58c5403b8cb21c29cfa778d3 2024-11-19T16:14:49,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/6a8a538b58c5403b8cb21c29cfa778d3, entries=150, sequenceid=62, filesize=30.2 K 2024-11-19T16:14:49,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/511c3ae536914114b3917c0ee8ad031a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/511c3ae536914114b3917c0ee8ad031a 2024-11-19T16:14:49,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/511c3ae536914114b3917c0ee8ad031a, entries=150, sequenceid=62, filesize=11.7 K 2024-11-19T16:14:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/0b701aec8c7f4cb7a4e20908f872361f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/0b701aec8c7f4cb7a4e20908f872361f 2024-11-19T16:14:49,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/0b701aec8c7f4cb7a4e20908f872361f, entries=150, sequenceid=62, filesize=11.7 K 2024-11-19T16:14:49,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 828823ece6f79c9635b39bcf387a8d6c in 578ms, sequenceid=62, compaction requested=true 2024-11-19T16:14:49,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:49,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:49,497 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:49,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:49,497 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:49,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:49,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:49,499 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:49,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:49,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:49,499 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/B is initiating minor compaction (all files) 2024-11-19T16:14:49,499 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/B in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:49,500 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/2b060cbea3f141ffb52499544d6ca75b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/65172e8e819e47c1b2d612e413f66ea3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/511c3ae536914114b3917c0ee8ad031a] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=35.2 K 2024-11-19T16:14:49,500 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:49,500 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/A is initiating minor compaction (all files) 2024-11-19T16:14:49,500 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/A in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:49,500 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42b75fa0ecae4a3f8c633d428fde5e6e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/06cf6b9f5aac43e09e7154b62bad6da9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/6a8a538b58c5403b8cb21c29cfa778d3] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=90.7 K 2024-11-19T16:14:49,500 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:49,500 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42b75fa0ecae4a3f8c633d428fde5e6e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/06cf6b9f5aac43e09e7154b62bad6da9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/6a8a538b58c5403b8cb21c29cfa778d3] 2024-11-19T16:14:49,501 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b060cbea3f141ffb52499544d6ca75b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732032887799 2024-11-19T16:14:49,501 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b75fa0ecae4a3f8c633d428fde5e6e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732032887799 2024-11-19T16:14:49,502 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 65172e8e819e47c1b2d612e413f66ea3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732032887910 2024-11-19T16:14:49,502 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 511c3ae536914114b3917c0ee8ad031a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732032888283 2024-11-19T16:14:49,502 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06cf6b9f5aac43e09e7154b62bad6da9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732032887910 2024-11-19T16:14:49,504 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a8a538b58c5403b8cb21c29cfa778d3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732032888283 2024-11-19T16:14:49,514 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#B#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:49,514 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/67bf2feb4c404ff390b170af9b946073 is 50, key is test_row_0/B:col10/1732032888283/Put/seqid=0 2024-11-19T16:14:49,533 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:49,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741966_1142 (size=12104) 2024-11-19T16:14:49,537 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119e15404ff30524973881a62f2e8fcbd04_828823ece6f79c9635b39bcf387a8d6c store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:49,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T16:14:49,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:49,546 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T16:14:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:14:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:14:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:14:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:49,550 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119e15404ff30524973881a62f2e8fcbd04_828823ece6f79c9635b39bcf387a8d6c, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:49,550 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e15404ff30524973881a62f2e8fcbd04_828823ece6f79c9635b39bcf387a8d6c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T16:14:49,563 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/67bf2feb4c404ff390b170af9b946073 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/67bf2feb4c404ff390b170af9b946073 2024-11-19T16:14:49,571 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/B of 828823ece6f79c9635b39bcf387a8d6c into 67bf2feb4c404ff390b170af9b946073(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:49,571 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:49,571 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/B, priority=13, startTime=1732032889497; duration=0sec 2024-11-19T16:14:49,571 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:49,571 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:B 2024-11-19T16:14:49,571 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:49,573 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:49,573 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/C is initiating minor compaction (all files) 2024-11-19T16:14:49,573 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/C in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:49,573 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/af7584d3d9b94ceebfac478b8d819116, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/46551c4f955749249a791aca1792a12e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/0b701aec8c7f4cb7a4e20908f872361f] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=35.2 K 2024-11-19T16:14:49,574 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting af7584d3d9b94ceebfac478b8d819116, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732032887799 2024-11-19T16:14:49,575 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 46551c4f955749249a791aca1792a12e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732032887910 2024-11-19T16:14:49,575 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b701aec8c7f4cb7a4e20908f872361f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732032888283 2024-11-19T16:14:49,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190727b07384ef4c1ea2741347b964614c_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032888948/Put/seqid=0 2024-11-19T16:14:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:49,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:49,599 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:49,599 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/e54e91fbbd2540d88e7b51c6e2b2a9e2 is 50, key is test_row_0/C:col10/1732032888283/Put/seqid=0 2024-11-19T16:14:49,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741968_1144 (size=12154) 2024-11-19T16:14:49,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032949608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032949609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032949610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032949613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032949614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032949716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741967_1143 (size=4469) 2024-11-19T16:14:49,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741969_1145 (size=12104) 2024-11-19T16:14:49,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032949727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032949727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,728 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#A#compaction#124 average throughput is 0.13 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032949727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032949727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,729 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/c0119c6b1cb1424eb36b61fd491269ec is 175, key is test_row_0/A:col10/1732032888283/Put/seqid=0 2024-11-19T16:14:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741970_1146 (size=31058) 2024-11-19T16:14:49,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032949927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032949937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032949937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032949938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032949938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:50,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:50,012 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190727b07384ef4c1ea2741347b964614c_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190727b07384ef4c1ea2741347b964614c_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/880dcc4350ae4e5c8f8b5636428a03f9, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:50,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/880dcc4350ae4e5c8f8b5636428a03f9 is 175, key is test_row_0/A:col10/1732032888948/Put/seqid=0 2024-11-19T16:14:50,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741971_1147 (size=30955) 2024-11-19T16:14:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T16:14:50,133 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/e54e91fbbd2540d88e7b51c6e2b2a9e2 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/e54e91fbbd2540d88e7b51c6e2b2a9e2 2024-11-19T16:14:50,148 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/C of 828823ece6f79c9635b39bcf387a8d6c into e54e91fbbd2540d88e7b51c6e2b2a9e2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:50,148 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:50,148 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/C, priority=13, startTime=1732032889499; duration=0sec 2024-11-19T16:14:50,148 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:50,148 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:C 2024-11-19T16:14:50,150 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/c0119c6b1cb1424eb36b61fd491269ec as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/c0119c6b1cb1424eb36b61fd491269ec 2024-11-19T16:14:50,162 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/A of 828823ece6f79c9635b39bcf387a8d6c into c0119c6b1cb1424eb36b61fd491269ec(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:50,162 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:50,162 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/A, priority=13, startTime=1732032889497; duration=0sec 2024-11-19T16:14:50,162 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:50,162 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:A 2024-11-19T16:14:50,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032950237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:50,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:50,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:50,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032950247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:50,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:50,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032950247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:50,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:50,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:50,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032950247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:50,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032950247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:50,333 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T16:14:50,420 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/880dcc4350ae4e5c8f8b5636428a03f9 2024-11-19T16:14:50,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/63d23a51ab1c42cfb327ead0ee41bd9f is 50, key is test_row_0/B:col10/1732032888948/Put/seqid=0 2024-11-19T16:14:50,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741972_1148 (size=12001) 2024-11-19T16:14:50,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032950747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:50,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032950757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:50,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032950757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:50,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032950766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:50,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032950766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:50,835 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/63d23a51ab1c42cfb327ead0ee41bd9f 2024-11-19T16:14:50,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/a55d27423b0a43b98ad415c8a992ff57 is 50, key is test_row_0/C:col10/1732032888948/Put/seqid=0 2024-11-19T16:14:50,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741973_1149 (size=12001) 2024-11-19T16:14:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T16:14:51,258 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/a55d27423b0a43b98ad415c8a992ff57 2024-11-19T16:14:51,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/880dcc4350ae4e5c8f8b5636428a03f9 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/880dcc4350ae4e5c8f8b5636428a03f9 2024-11-19T16:14:51,276 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/880dcc4350ae4e5c8f8b5636428a03f9, entries=150, sequenceid=81, filesize=30.2 K 2024-11-19T16:14:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/63d23a51ab1c42cfb327ead0ee41bd9f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/63d23a51ab1c42cfb327ead0ee41bd9f 2024-11-19T16:14:51,283 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/63d23a51ab1c42cfb327ead0ee41bd9f, entries=150, sequenceid=81, filesize=11.7 K 2024-11-19T16:14:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/a55d27423b0a43b98ad415c8a992ff57 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a55d27423b0a43b98ad415c8a992ff57 2024-11-19T16:14:51,291 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a55d27423b0a43b98ad415c8a992ff57, entries=150, sequenceid=81, filesize=11.7 K 2024-11-19T16:14:51,293 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 828823ece6f79c9635b39bcf387a8d6c in 1746ms, sequenceid=81, compaction requested=false 2024-11-19T16:14:51,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:51,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:51,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-19T16:14:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-19T16:14:51,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-19T16:14:51,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3630 sec 2024-11-19T16:14:51,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.3690 sec 2024-11-19T16:14:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:51,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T16:14:51,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:14:51,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:51,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:14:51,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:51,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:14:51,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:51,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ed71c66b2bfa47b09ad12b8bd68261ac_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032891767/Put/seqid=0 2024-11-19T16:14:51,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741974_1150 (size=17034) 2024-11-19T16:14:51,782 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:51,792 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ed71c66b2bfa47b09ad12b8bd68261ac_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ed71c66b2bfa47b09ad12b8bd68261ac_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:51,793 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/d275d6b0c62d465795cfedf6fea4f603, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:51,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/d275d6b0c62d465795cfedf6fea4f603 is 175, key is test_row_0/A:col10/1732032891767/Put/seqid=0 2024-11-19T16:14:51,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032951788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:51,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032951791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:51,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:51,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032951794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:51,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032951796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:51,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032951796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:51,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741975_1151 (size=48139) 2024-11-19T16:14:51,829 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/d275d6b0c62d465795cfedf6fea4f603 2024-11-19T16:14:51,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/e7afecfafb4b4c728ffa551351e0703e is 50, key is test_row_0/B:col10/1732032891767/Put/seqid=0 2024-11-19T16:14:51,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741976_1152 (size=12001) 2024-11-19T16:14:51,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/e7afecfafb4b4c728ffa551351e0703e 2024-11-19T16:14:51,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/86e11f7fa8054edab36e2727fe621a23 is 50, key is test_row_0/C:col10/1732032891767/Put/seqid=0 2024-11-19T16:14:51,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741977_1153 (size=12001) 2024-11-19T16:14:51,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/86e11f7fa8054edab36e2727fe621a23 2024-11-19T16:14:51,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/d275d6b0c62d465795cfedf6fea4f603 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/d275d6b0c62d465795cfedf6fea4f603 2024-11-19T16:14:51,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/d275d6b0c62d465795cfedf6fea4f603, entries=250, sequenceid=99, filesize=47.0 K 2024-11-19T16:14:51,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/e7afecfafb4b4c728ffa551351e0703e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/e7afecfafb4b4c728ffa551351e0703e 2024-11-19T16:14:51,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/e7afecfafb4b4c728ffa551351e0703e, entries=150, sequenceid=99, filesize=11.7 K 2024-11-19T16:14:51,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/86e11f7fa8054edab36e2727fe621a23 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/86e11f7fa8054edab36e2727fe621a23 2024-11-19T16:14:51,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/86e11f7fa8054edab36e2727fe621a23, entries=150, sequenceid=99, filesize=11.7 K 2024-11-19T16:14:51,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 828823ece6f79c9635b39bcf387a8d6c in 126ms, sequenceid=99, compaction requested=true 2024-11-19T16:14:51,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:51,896 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:51,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:51,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:51,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:51,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:51,896 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:51,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:51,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:51,897 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:51,897 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/A is initiating minor compaction (all files) 2024-11-19T16:14:51,897 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/A in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:51,897 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/c0119c6b1cb1424eb36b61fd491269ec, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/880dcc4350ae4e5c8f8b5636428a03f9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/d275d6b0c62d465795cfedf6fea4f603] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=107.6 K 2024-11-19T16:14:51,897 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:51,897 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/c0119c6b1cb1424eb36b61fd491269ec, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/880dcc4350ae4e5c8f8b5636428a03f9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/d275d6b0c62d465795cfedf6fea4f603] 2024-11-19T16:14:51,899 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:51,899 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0119c6b1cb1424eb36b61fd491269ec, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732032888283 2024-11-19T16:14:51,899 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/B is initiating minor compaction (all files) 2024-11-19T16:14:51,899 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/B in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:51,899 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/67bf2feb4c404ff390b170af9b946073, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/63d23a51ab1c42cfb327ead0ee41bd9f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/e7afecfafb4b4c728ffa551351e0703e] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=35.3 K 2024-11-19T16:14:51,900 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 880dcc4350ae4e5c8f8b5636428a03f9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732032888948 2024-11-19T16:14:51,900 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 67bf2feb4c404ff390b170af9b946073, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732032888283 2024-11-19T16:14:51,901 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 63d23a51ab1c42cfb327ead0ee41bd9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732032888948 2024-11-19T16:14:51,901 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting d275d6b0c62d465795cfedf6fea4f603, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732032889605 2024-11-19T16:14:51,902 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting e7afecfafb4b4c728ffa551351e0703e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732032889612 2024-11-19T16:14:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:51,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T16:14:51,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:14:51,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:51,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:14:51,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:51,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:14:51,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:51,913 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:51,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032951923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:51,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:51,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032951925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:51,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:51,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,930 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032951925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:51,930 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/6ca58ada417c4e419110a016f8ded5f1 is 50, key is test_row_0/B:col10/1732032891767/Put/seqid=0 2024-11-19T16:14:51,933 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411199b0d62599d774b01ab6e2d620f0f8f75_828823ece6f79c9635b39bcf387a8d6c store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:51,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032951927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:51,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032951929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:51,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:51,938 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411199b0d62599d774b01ab6e2d620f0f8f75_828823ece6f79c9635b39bcf387a8d6c, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:51,938 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199b0d62599d774b01ab6e2d620f0f8f75_828823ece6f79c9635b39bcf387a8d6c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:51,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119795ff68a42b34522ac22798e95b40a61_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032891788/Put/seqid=0 2024-11-19T16:14:51,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741978_1154 (size=12207) 2024-11-19T16:14:51,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741979_1155 (size=4469) 2024-11-19T16:14:51,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741980_1156 (size=14594) 2024-11-19T16:14:51,972 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:51,977 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/6ca58ada417c4e419110a016f8ded5f1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/6ca58ada417c4e419110a016f8ded5f1 2024-11-19T16:14:51,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119795ff68a42b34522ac22798e95b40a61_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119795ff68a42b34522ac22798e95b40a61_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:51,982 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/e8112b06b22440aebce1d80cd9ed2591, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:51,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/e8112b06b22440aebce1d80cd9ed2591 is 175, key is test_row_0/A:col10/1732032891788/Put/seqid=0 2024-11-19T16:14:51,993 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/B of 828823ece6f79c9635b39bcf387a8d6c into 6ca58ada417c4e419110a016f8ded5f1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:51,993 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:51,993 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/B, priority=13, startTime=1732032891896; duration=0sec 2024-11-19T16:14:51,993 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:51,993 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:B 2024-11-19T16:14:51,993 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:51,995 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:51,995 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/C is initiating minor compaction (all files) 2024-11-19T16:14:51,995 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/C in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:51,996 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/e54e91fbbd2540d88e7b51c6e2b2a9e2, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a55d27423b0a43b98ad415c8a992ff57, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/86e11f7fa8054edab36e2727fe621a23] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=35.3 K 2024-11-19T16:14:51,996 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting e54e91fbbd2540d88e7b51c6e2b2a9e2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732032888283 2024-11-19T16:14:52,000 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a55d27423b0a43b98ad415c8a992ff57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732032888948 2024-11-19T16:14:52,000 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 86e11f7fa8054edab36e2727fe621a23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732032889612 2024-11-19T16:14:52,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741981_1157 (size=39549) 2024-11-19T16:14:52,014 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:52,015 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/52aea822467a447f9c5dd67b36ba6b08 is 50, key is test_row_0/C:col10/1732032891767/Put/seqid=0 2024-11-19T16:14:52,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741982_1158 (size=12207) 2024-11-19T16:14:52,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032952037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032952037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032952037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032952047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032952047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,056 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/52aea822467a447f9c5dd67b36ba6b08 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/52aea822467a447f9c5dd67b36ba6b08 2024-11-19T16:14:52,070 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/C of 828823ece6f79c9635b39bcf387a8d6c into 52aea822467a447f9c5dd67b36ba6b08(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:52,070 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:52,070 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/C, priority=13, startTime=1732032891896; duration=0sec 2024-11-19T16:14:52,070 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:52,070 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:C 2024-11-19T16:14:52,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032952246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032952247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032952247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032952257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032952257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,366 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#A#compaction#132 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:52,367 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/fd08ef51c0da4d5db9de4c2a586164a8 is 175, key is test_row_0/A:col10/1732032891767/Put/seqid=0 2024-11-19T16:14:52,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741983_1159 (size=31161) 2024-11-19T16:14:52,413 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/e8112b06b22440aebce1d80cd9ed2591 2024-11-19T16:14:52,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/e776e866585b4b1ca8dcd28e79ff21ca is 50, key is test_row_0/B:col10/1732032891788/Put/seqid=0 2024-11-19T16:14:52,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741984_1160 (size=12001) 2024-11-19T16:14:52,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032952556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032952556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032952556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032952567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032952567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:52,775 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/fd08ef51c0da4d5db9de4c2a586164a8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/fd08ef51c0da4d5db9de4c2a586164a8 2024-11-19T16:14:52,781 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/A of 828823ece6f79c9635b39bcf387a8d6c into fd08ef51c0da4d5db9de4c2a586164a8(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:52,781 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:52,781 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/A, priority=13, startTime=1732032891895; duration=0sec 2024-11-19T16:14:52,781 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:52,781 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:A 2024-11-19T16:14:52,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/e776e866585b4b1ca8dcd28e79ff21ca 2024-11-19T16:14:52,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/9a896c983d2e4f7a96e92c04f89757fe is 50, key is test_row_0/C:col10/1732032891788/Put/seqid=0 2024-11-19T16:14:52,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741985_1161 (size=12001) 2024-11-19T16:14:53,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032953067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:53,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032953067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:53,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:53,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:53,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032953067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:53,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:53,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:53,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:53,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032953076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:53,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032953076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T16:14:53,097 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:53,097 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-19T16:14:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T16:14:53,099 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:53,100 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:53,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T16:14:53,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/9a896c983d2e4f7a96e92c04f89757fe 2024-11-19T16:14:53,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T16:14:53,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:53,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:53,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:53,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:53,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:53,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:53,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/e8112b06b22440aebce1d80cd9ed2591 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/e8112b06b22440aebce1d80cd9ed2591 2024-11-19T16:14:53,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/e8112b06b22440aebce1d80cd9ed2591, entries=200, sequenceid=117, filesize=38.6 K 2024-11-19T16:14:53,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/e776e866585b4b1ca8dcd28e79ff21ca as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/e776e866585b4b1ca8dcd28e79ff21ca 2024-11-19T16:14:53,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/e776e866585b4b1ca8dcd28e79ff21ca, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T16:14:53,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/9a896c983d2e4f7a96e92c04f89757fe as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/9a896c983d2e4f7a96e92c04f89757fe 2024-11-19T16:14:53,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/9a896c983d2e4f7a96e92c04f89757fe, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T16:14:53,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 828823ece6f79c9635b39bcf387a8d6c in 1362ms, sequenceid=117, compaction requested=false 2024-11-19T16:14:53,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:53,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T16:14:53,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:53,406 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T16:14:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:14:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:14:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:14:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:53,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111952e02d371035475bbf54235f2fbbbf09_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032891922/Put/seqid=0 2024-11-19T16:14:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T16:14:53,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741986_1162 (size=12254) 2024-11-19T16:14:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T16:14:53,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:53,851 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111952e02d371035475bbf54235f2fbbbf09_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111952e02d371035475bbf54235f2fbbbf09_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:53,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/692f7f5a4eaa473f890ff43ba7b79e4f, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:53,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/692f7f5a4eaa473f890ff43ba7b79e4f is 175, key is test_row_0/A:col10/1732032891922/Put/seqid=0 2024-11-19T16:14:53,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741987_1163 (size=31055) 2024-11-19T16:14:53,879 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/692f7f5a4eaa473f890ff43ba7b79e4f 2024-11-19T16:14:53,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/1dd568beff7347e99d34265d6b8c8941 is 50, key is test_row_0/B:col10/1732032891922/Put/seqid=0 2024-11-19T16:14:53,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741988_1164 (size=12101) 2024-11-19T16:14:53,908 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/1dd568beff7347e99d34265d6b8c8941 2024-11-19T16:14:53,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/f1c13857d8784cfaafa4d2fbe1af9d0f is 50, key is test_row_0/C:col10/1732032891922/Put/seqid=0 2024-11-19T16:14:53,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741989_1165 (size=12101) 2024-11-19T16:14:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:54,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:54,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032954101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032954102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032954103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032954104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032954108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032954217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032954217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032954218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032954218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032954218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T16:14:54,327 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/f1c13857d8784cfaafa4d2fbe1af9d0f 2024-11-19T16:14:54,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/692f7f5a4eaa473f890ff43ba7b79e4f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/692f7f5a4eaa473f890ff43ba7b79e4f 2024-11-19T16:14:54,342 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/692f7f5a4eaa473f890ff43ba7b79e4f, entries=150, sequenceid=134, filesize=30.3 K 2024-11-19T16:14:54,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/1dd568beff7347e99d34265d6b8c8941 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/1dd568beff7347e99d34265d6b8c8941 2024-11-19T16:14:54,350 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/1dd568beff7347e99d34265d6b8c8941, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T16:14:54,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/f1c13857d8784cfaafa4d2fbe1af9d0f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/f1c13857d8784cfaafa4d2fbe1af9d0f 2024-11-19T16:14:54,363 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/f1c13857d8784cfaafa4d2fbe1af9d0f, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T16:14:54,365 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 828823ece6f79c9635b39bcf387a8d6c in 960ms, sequenceid=134, compaction requested=true 2024-11-19T16:14:54,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:54,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:54,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-19T16:14:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-19T16:14:54,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-19T16:14:54,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2670 sec 2024-11-19T16:14:54,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.2720 sec 2024-11-19T16:14:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:54,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T16:14:54,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:14:54,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:54,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:14:54,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:54,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:14:54,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:54,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111970551f9eed004330b72a77be7f8c7eb5_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032894429/Put/seqid=0 2024-11-19T16:14:54,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741990_1166 (size=12304) 2024-11-19T16:14:54,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032954443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032954449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032954450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032954450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032954451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032954556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032954556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032954557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032954557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032954567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032954766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032954766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032954766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032954767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032954777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:54,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:54,854 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111970551f9eed004330b72a77be7f8c7eb5_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111970551f9eed004330b72a77be7f8c7eb5_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:54,855 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/58878f6d3f634eb98b99d300a3c05e74, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:54,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/58878f6d3f634eb98b99d300a3c05e74 is 175, key is test_row_0/A:col10/1732032894429/Put/seqid=0 2024-11-19T16:14:54,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741991_1167 (size=31105) 2024-11-19T16:14:54,859 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/58878f6d3f634eb98b99d300a3c05e74 2024-11-19T16:14:54,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/792b374e18e646efb8ac8de63a24f6f4 is 50, key is test_row_0/B:col10/1732032894429/Put/seqid=0 2024-11-19T16:14:54,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741992_1168 (size=12151) 2024-11-19T16:14:55,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032955076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:55,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032955077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:55,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032955077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:55,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032955077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:55,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032955087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T16:14:55,256 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:55,257 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-19T16:14:55,259 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T16:14:55,259 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:55,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:55,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/792b374e18e646efb8ac8de63a24f6f4 2024-11-19T16:14:55,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/aa76bd7eed5a41a0952887a0a3f90219 is 50, key is test_row_0/C:col10/1732032894429/Put/seqid=0 2024-11-19T16:14:55,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741993_1169 (size=12151) 2024-11-19T16:14:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T16:14:55,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T16:14:55,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:55,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:55,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:55,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:55,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:55,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T16:14:55,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:55,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:55,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:55,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T16:14:55,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032955596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:55,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032955597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:55,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032955597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032955597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032955597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:55,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/aa76bd7eed5a41a0952887a0a3f90219 2024-11-19T16:14:55,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/58878f6d3f634eb98b99d300a3c05e74 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/58878f6d3f634eb98b99d300a3c05e74 2024-11-19T16:14:55,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/58878f6d3f634eb98b99d300a3c05e74, entries=150, sequenceid=155, filesize=30.4 K 2024-11-19T16:14:55,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/792b374e18e646efb8ac8de63a24f6f4 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/792b374e18e646efb8ac8de63a24f6f4 2024-11-19T16:14:55,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/792b374e18e646efb8ac8de63a24f6f4, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T16:14:55,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/aa76bd7eed5a41a0952887a0a3f90219 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/aa76bd7eed5a41a0952887a0a3f90219 2024-11-19T16:14:55,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/aa76bd7eed5a41a0952887a0a3f90219, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T16:14:55,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 828823ece6f79c9635b39bcf387a8d6c in 1281ms, sequenceid=155, compaction requested=true 2024-11-19T16:14:55,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:55,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:55,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:55,712 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:55,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:55,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:55,712 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:55,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:55,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:55,713 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132870 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:55,713 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:55,714 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/B is initiating minor compaction (all files) 2024-11-19T16:14:55,714 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/A is initiating minor compaction (all files) 2024-11-19T16:14:55,714 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/A in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:55,714 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/B in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:55,714 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/6ca58ada417c4e419110a016f8ded5f1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/e776e866585b4b1ca8dcd28e79ff21ca, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/1dd568beff7347e99d34265d6b8c8941, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/792b374e18e646efb8ac8de63a24f6f4] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=47.3 K 2024-11-19T16:14:55,714 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/fd08ef51c0da4d5db9de4c2a586164a8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/e8112b06b22440aebce1d80cd9ed2591, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/692f7f5a4eaa473f890ff43ba7b79e4f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/58878f6d3f634eb98b99d300a3c05e74] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=129.8 K 2024-11-19T16:14:55,714 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:55,714 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/fd08ef51c0da4d5db9de4c2a586164a8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/e8112b06b22440aebce1d80cd9ed2591, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/692f7f5a4eaa473f890ff43ba7b79e4f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/58878f6d3f634eb98b99d300a3c05e74] 2024-11-19T16:14:55,714 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd08ef51c0da4d5db9de4c2a586164a8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732032889612 2024-11-19T16:14:55,715 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ca58ada417c4e419110a016f8ded5f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732032889612 2024-11-19T16:14:55,715 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8112b06b22440aebce1d80cd9ed2591, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732032891788 2024-11-19T16:14:55,715 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting e776e866585b4b1ca8dcd28e79ff21ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732032891788 2024-11-19T16:14:55,715 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 692f7f5a4eaa473f890ff43ba7b79e4f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732032891922 2024-11-19T16:14:55,715 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dd568beff7347e99d34265d6b8c8941, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732032891922 2024-11-19T16:14:55,716 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58878f6d3f634eb98b99d300a3c05e74, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732032894094 2024-11-19T16:14:55,716 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 792b374e18e646efb8ac8de63a24f6f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732032894094 2024-11-19T16:14:55,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T16:14:55,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:55,717 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T16:14:55,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:14:55,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:55,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:14:55,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:55,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:14:55,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:55,729 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:55,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194700052d96ba479d939654b847222be8_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032894448/Put/seqid=0 2024-11-19T16:14:55,730 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:55,730 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/f5e124906efd47e58394725dad20b0f1 is 50, key is test_row_0/B:col10/1732032894429/Put/seqid=0 2024-11-19T16:14:55,731 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119d1956d327c3348dd837a53a6f8bf18b4_828823ece6f79c9635b39bcf387a8d6c store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:55,734 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119d1956d327c3348dd837a53a6f8bf18b4_828823ece6f79c9635b39bcf387a8d6c, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:55,735 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d1956d327c3348dd837a53a6f8bf18b4_828823ece6f79c9635b39bcf387a8d6c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:55,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741995_1171 (size=12493) 2024-11-19T16:14:55,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741994_1170 (size=12304) 2024-11-19T16:14:55,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:55,767 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194700052d96ba479d939654b847222be8_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194700052d96ba479d939654b847222be8_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:55,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/c3103081ba694924b48f7ad7bcd3d587, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:55,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/c3103081ba694924b48f7ad7bcd3d587 is 175, key is test_row_0/A:col10/1732032894448/Put/seqid=0 2024-11-19T16:14:55,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741996_1172 (size=4469) 2024-11-19T16:14:55,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741997_1173 (size=31105) 2024-11-19T16:14:55,790 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/c3103081ba694924b48f7ad7bcd3d587 2024-11-19T16:14:55,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/41a6f0fe506d40529d11d2a6d3f7441a is 50, key is test_row_0/B:col10/1732032894448/Put/seqid=0 2024-11-19T16:14:55,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741998_1174 (size=12151) 2024-11-19T16:14:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T16:14:56,161 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/f5e124906efd47e58394725dad20b0f1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f5e124906efd47e58394725dad20b0f1 2024-11-19T16:14:56,166 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/B of 828823ece6f79c9635b39bcf387a8d6c into f5e124906efd47e58394725dad20b0f1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:56,167 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:56,167 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/B, priority=12, startTime=1732032895712; duration=0sec 2024-11-19T16:14:56,167 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:56,167 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:B 2024-11-19T16:14:56,167 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:14:56,168 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:14:56,168 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/C is initiating minor compaction (all files) 2024-11-19T16:14:56,168 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/C in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:56,168 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/52aea822467a447f9c5dd67b36ba6b08, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/9a896c983d2e4f7a96e92c04f89757fe, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/f1c13857d8784cfaafa4d2fbe1af9d0f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/aa76bd7eed5a41a0952887a0a3f90219] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=47.3 K 2024-11-19T16:14:56,168 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 52aea822467a447f9c5dd67b36ba6b08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732032889612 2024-11-19T16:14:56,169 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a896c983d2e4f7a96e92c04f89757fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732032891788 2024-11-19T16:14:56,169 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting f1c13857d8784cfaafa4d2fbe1af9d0f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732032891922 2024-11-19T16:14:56,169 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting aa76bd7eed5a41a0952887a0a3f90219, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732032894094 2024-11-19T16:14:56,178 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#C#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:56,178 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/a7318b053b98411d96e0e1729ea37663 is 50, key is test_row_0/C:col10/1732032894429/Put/seqid=0 2024-11-19T16:14:56,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741999_1175 (size=12493) 2024-11-19T16:14:56,184 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#A#compaction#146 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:56,184 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/ec0424dc21864c21bd454d2446228a3a is 175, key is test_row_0/A:col10/1732032894429/Put/seqid=0 2024-11-19T16:14:56,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742000_1176 (size=31447) 2024-11-19T16:14:56,214 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/41a6f0fe506d40529d11d2a6d3f7441a 2024-11-19T16:14:56,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/995cd29ae13f4887936aaab9eabede03 is 50, key is test_row_0/C:col10/1732032894448/Put/seqid=0 2024-11-19T16:14:56,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742001_1177 (size=12151) 2024-11-19T16:14:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T16:14:56,587 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/a7318b053b98411d96e0e1729ea37663 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a7318b053b98411d96e0e1729ea37663 2024-11-19T16:14:56,593 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/ec0424dc21864c21bd454d2446228a3a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/ec0424dc21864c21bd454d2446228a3a 2024-11-19T16:14:56,594 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/C of 828823ece6f79c9635b39bcf387a8d6c into a7318b053b98411d96e0e1729ea37663(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:56,594 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:56,594 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/C, priority=12, startTime=1732032895712; duration=0sec 2024-11-19T16:14:56,594 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:56,594 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:C 2024-11-19T16:14:56,599 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/A of 828823ece6f79c9635b39bcf387a8d6c into ec0424dc21864c21bd454d2446228a3a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:56,599 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:56,599 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/A, priority=12, startTime=1732032895712; duration=0sec 2024-11-19T16:14:56,599 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:56,599 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:A 2024-11-19T16:14:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:56,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:56,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032956627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:56,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032956628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:56,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:56,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032896625, max timestamp=1732032896630Num uniques 859; 2024-11-19T16:14:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032956630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:56,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:14:56,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:56,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032896625, max timestamp=1732032896630Num uniques 859; 2024-11-19T16:14:56,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:14:56,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:56,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032896625, max timestamp=1732032896630Num uniques 859; 2024-11-19T16:14:56,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:14:56,636 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/995cd29ae13f4887936aaab9eabede03 2024-11-19T16:14:56,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/c3103081ba694924b48f7ad7bcd3d587 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/c3103081ba694924b48f7ad7bcd3d587 2024-11-19T16:14:56,650 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/c3103081ba694924b48f7ad7bcd3d587, entries=150, sequenceid=167, filesize=30.4 K 2024-11-19T16:14:56,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/41a6f0fe506d40529d11d2a6d3f7441a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/41a6f0fe506d40529d11d2a6d3f7441a 2024-11-19T16:14:56,656 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/41a6f0fe506d40529d11d2a6d3f7441a, entries=150, sequenceid=167, filesize=11.9 K 2024-11-19T16:14:56,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/995cd29ae13f4887936aaab9eabede03 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/995cd29ae13f4887936aaab9eabede03 2024-11-19T16:14:56,664 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/995cd29ae13f4887936aaab9eabede03, entries=150, sequenceid=167, filesize=11.9 K 2024-11-19T16:14:56,666 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=89.45 KB/91600 for 828823ece6f79c9635b39bcf387a8d6c in 949ms, sequenceid=167, compaction requested=false 2024-11-19T16:14:56,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:56,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:56,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-19T16:14:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:56,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=96.16 KB heapSize=265.84 KB 2024-11-19T16:14:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-19T16:14:56,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:14:56,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:56,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:14:56,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:56,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:14:56,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:56,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-19T16:14:56,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-19T16:14:56,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4150 sec 2024-11-19T16:14:56,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119436413e77cc5400f8ef3275468fbedcf_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032896645/Put/seqid=0 2024-11-19T16:14:56,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742002_1178 (size=17284) 2024-11-19T16:14:56,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032956709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:56,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032956715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:56,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032956736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:56,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032956736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:56,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:56,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032956736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:56,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:56,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:56,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032956826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:56,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032956826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:56,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032956946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:56,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032956947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:56,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032956947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:57,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032957036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:57,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032957037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:57,102 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:57,106 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119436413e77cc5400f8ef3275468fbedcf_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119436413e77cc5400f8ef3275468fbedcf_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:57,107 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/2750b1bdeb4b4f43b6ee614f73154085, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:57,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/2750b1bdeb4b4f43b6ee614f73154085 is 175, key is test_row_0/A:col10/1732032896645/Put/seqid=0 2024-11-19T16:14:57,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742003_1179 (size=48389) 2024-11-19T16:14:57,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032957256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:57,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032957256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:57,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032957257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:57,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032957346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:57,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032957346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T16:14:57,407 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:57,408 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-19T16:14:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T16:14:57,409 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:57,410 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:57,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:57,511 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/2750b1bdeb4b4f43b6ee614f73154085 2024-11-19T16:14:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T16:14:57,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/d0743313b39449b7a21da845a544ed61 is 50, key is test_row_0/B:col10/1732032896645/Put/seqid=0 2024-11-19T16:14:57,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742004_1180 (size=12151) 2024-11-19T16:14:57,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T16:14:57,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:57,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:57,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:57,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:57,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:57,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T16:14:57,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:57,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:57,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:57,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:57,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T16:14:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:57,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032957767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032957767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032957767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:57,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032957857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:57,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:57,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032957867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:57,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:57,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:57,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T16:14:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:57,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:57,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:57,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/d0743313b39449b7a21da845a544ed61 2024-11-19T16:14:57,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/c10ec72ea0514b9f81c66ea5882f4c6d is 50, key is test_row_0/C:col10/1732032896645/Put/seqid=0 2024-11-19T16:14:57,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742005_1181 (size=12151) 2024-11-19T16:14:57,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/c10ec72ea0514b9f81c66ea5882f4c6d 2024-11-19T16:14:57,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/2750b1bdeb4b4f43b6ee614f73154085 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/2750b1bdeb4b4f43b6ee614f73154085 2024-11-19T16:14:57,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/2750b1bdeb4b4f43b6ee614f73154085, entries=250, sequenceid=200, filesize=47.3 K 2024-11-19T16:14:57,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/d0743313b39449b7a21da845a544ed61 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/d0743313b39449b7a21da845a544ed61 2024-11-19T16:14:57,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/d0743313b39449b7a21da845a544ed61, entries=150, sequenceid=200, filesize=11.9 K 2024-11-19T16:14:57,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/c10ec72ea0514b9f81c66ea5882f4c6d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/c10ec72ea0514b9f81c66ea5882f4c6d 2024-11-19T16:14:57,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/c10ec72ea0514b9f81c66ea5882f4c6d, entries=150, sequenceid=200, filesize=11.9 K 2024-11-19T16:14:57,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~265.80 KB/272176, currentSize=87.22 KB/89310 for 828823ece6f79c9635b39bcf387a8d6c in 1332ms, sequenceid=200, compaction requested=true 2024-11-19T16:14:57,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:57,998 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:57,999 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:57,999 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/A is initiating minor compaction (all files) 2024-11-19T16:14:58,000 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/A in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:58,000 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/ec0424dc21864c21bd454d2446228a3a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/c3103081ba694924b48f7ad7bcd3d587, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/2750b1bdeb4b4f43b6ee614f73154085] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=108.3 K 2024-11-19T16:14:58,000 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:58,000 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/ec0424dc21864c21bd454d2446228a3a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/c3103081ba694924b48f7ad7bcd3d587, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/2750b1bdeb4b4f43b6ee614f73154085] 2024-11-19T16:14:58,001 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec0424dc21864c21bd454d2446228a3a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732032894094 2024-11-19T16:14:58,001 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3103081ba694924b48f7ad7bcd3d587, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732032894446 2024-11-19T16:14:58,002 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2750b1bdeb4b4f43b6ee614f73154085, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732032896645 2024-11-19T16:14:58,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:58,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:58,003 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:58,005 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:58,005 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/B is initiating minor compaction (all files) 2024-11-19T16:14:58,005 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/B in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:58,005 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f5e124906efd47e58394725dad20b0f1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/41a6f0fe506d40529d11d2a6d3f7441a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/d0743313b39449b7a21da845a544ed61] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=35.9 K 2024-11-19T16:14:58,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:58,006 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting f5e124906efd47e58394725dad20b0f1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732032894094 2024-11-19T16:14:58,006 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a6f0fe506d40529d11d2a6d3f7441a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732032894446 2024-11-19T16:14:58,008 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting d0743313b39449b7a21da845a544ed61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732032896645 2024-11-19T16:14:58,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:58,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:58,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:58,022 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:58,031 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#B#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:58,032 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/28ef2dcc408d46ecac3088175e91a891 is 50, key is test_row_0/B:col10/1732032896645/Put/seqid=0 2024-11-19T16:14:58,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T16:14:58,033 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411197a11f9cea74a42c581fdad5702e5ff66_828823ece6f79c9635b39bcf387a8d6c store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:58,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:58,034 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T16:14:58,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:14:58,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:58,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:14:58,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:58,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:14:58,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:58,036 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411197a11f9cea74a42c581fdad5702e5ff66_828823ece6f79c9635b39bcf387a8d6c, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:58,036 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197a11f9cea74a42c581fdad5702e5ff66_828823ece6f79c9635b39bcf387a8d6c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T16:14:58,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742006_1182 (size=12595) 2024-11-19T16:14:58,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742007_1183 (size=4469) 2024-11-19T16:14:58,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119681543d23b2545fe8f158ad9ee56a211_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032896703/Put/seqid=0 2024-11-19T16:14:58,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742008_1184 (size=12304) 2024-11-19T16:14:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,182 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119681543d23b2545fe8f158ad9ee56a211_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119681543d23b2545fe8f158ad9ee56a211_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:58,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/66359ac01aee4832b71ced0993a8c304, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:58,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/66359ac01aee4832b71ced0993a8c304 is 175, key is test_row_0/A:col10/1732032896703/Put/seqid=0 2024-11-19T16:14:58,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742009_1185 (size=31105) 2024-11-19T16:14:58,196 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/66359ac01aee4832b71ced0993a8c304 2024-11-19T16:14:58,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/c434ac7d856f42a184837f44be5c3e45 is 50, key is test_row_0/B:col10/1732032896703/Put/seqid=0 2024-11-19T16:14:58,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742010_1186 (size=12151) 2024-11-19T16:14:58,510 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/28ef2dcc408d46ecac3088175e91a891 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/28ef2dcc408d46ecac3088175e91a891 2024-11-19T16:14:58,515 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/B of 828823ece6f79c9635b39bcf387a8d6c into 28ef2dcc408d46ecac3088175e91a891(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:58,515 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:58,515 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/B, priority=13, startTime=1732032898003; duration=0sec 2024-11-19T16:14:58,515 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:58,515 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:B 2024-11-19T16:14:58,516 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:58,516 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:58,516 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/C is initiating minor compaction (all files) 2024-11-19T16:14:58,517 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/C in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:58,517 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a7318b053b98411d96e0e1729ea37663, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/995cd29ae13f4887936aaab9eabede03, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/c10ec72ea0514b9f81c66ea5882f4c6d] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=35.9 K 2024-11-19T16:14:58,517 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a7318b053b98411d96e0e1729ea37663, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732032894094 2024-11-19T16:14:58,517 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 995cd29ae13f4887936aaab9eabede03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732032894446 2024-11-19T16:14:58,517 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting c10ec72ea0514b9f81c66ea5882f4c6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732032896645 2024-11-19T16:14:58,526 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#C#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:58,527 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/43b8c5b061e64bd0a22f391f158fffc1 is 50, key is test_row_0/C:col10/1732032896645/Put/seqid=0 2024-11-19T16:14:58,536 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#A#compaction#153 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:58,537 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/d10f5b653b95453ba64834e48416cd6c is 175, key is test_row_0/A:col10/1732032896645/Put/seqid=0 2024-11-19T16:14:58,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742011_1187 (size=12595) 2024-11-19T16:14:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T16:14:58,550 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/43b8c5b061e64bd0a22f391f158fffc1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/43b8c5b061e64bd0a22f391f158fffc1 2024-11-19T16:14:58,555 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/C of 828823ece6f79c9635b39bcf387a8d6c into 43b8c5b061e64bd0a22f391f158fffc1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:58,555 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:58,555 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/C, priority=13, startTime=1732032898017; duration=0sec 2024-11-19T16:14:58,555 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:58,555 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:C 2024-11-19T16:14:58,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742012_1188 (size=31549) 2024-11-19T16:14:58,570 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/d10f5b653b95453ba64834e48416cd6c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/d10f5b653b95453ba64834e48416cd6c 2024-11-19T16:14:58,576 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/A of 828823ece6f79c9635b39bcf387a8d6c into d10f5b653b95453ba64834e48416cd6c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:58,576 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:58,576 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/A, priority=13, startTime=1732032897998; duration=0sec 2024-11-19T16:14:58,577 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:58,577 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:A 2024-11-19T16:14:58,629 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/c434ac7d856f42a184837f44be5c3e45 2024-11-19T16:14:58,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/383e7f1417d44006ae433747fbcf8630 is 50, key is test_row_0/C:col10/1732032896703/Put/seqid=0 2024-11-19T16:14:58,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742013_1189 (size=12151) 2024-11-19T16:14:58,660 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/383e7f1417d44006ae433747fbcf8630 2024-11-19T16:14:58,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/66359ac01aee4832b71ced0993a8c304 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/66359ac01aee4832b71ced0993a8c304 2024-11-19T16:14:58,676 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/66359ac01aee4832b71ced0993a8c304, entries=150, sequenceid=216, filesize=30.4 K 2024-11-19T16:14:58,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/c434ac7d856f42a184837f44be5c3e45 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/c434ac7d856f42a184837f44be5c3e45 2024-11-19T16:14:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,681 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/c434ac7d856f42a184837f44be5c3e45, entries=150, sequenceid=216, filesize=11.9 K 2024-11-19T16:14:58,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/383e7f1417d44006ae433747fbcf8630 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/383e7f1417d44006ae433747fbcf8630 2024-11-19T16:14:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,691 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/383e7f1417d44006ae433747fbcf8630, entries=150, sequenceid=216, filesize=11.9 K 2024-11-19T16:14:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,692 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 828823ece6f79c9635b39bcf387a8d6c in 658ms, sequenceid=216, compaction requested=false 2024-11-19T16:14:58,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:58,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:58,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-19T16:14:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-19T16:14:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-19T16:14:58,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2860 sec 2024-11-19T16:14:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.2910 sec 2024-11-19T16:14:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:58,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:14:58,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:14:58,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:58,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:14:58,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:58,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:14:58,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:14:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f11d1f13ea174a3cacb39b2aecfbc975_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032898853/Put/seqid=0 2024-11-19T16:14:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742014_1190 (size=22268) 2024-11-19T16:14:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39334, totalHeapSize=89576, min timestamp=1732032898856, max timestamp=1732032898908Num uniques 859; 2024-11-19T16:14:58,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39334, totalHeapSize=89576, min timestamp=1732032898856, max timestamp=1732032898908Num uniques 859; 2024-11-19T16:14:58,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=39334, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-19T16:14:58,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=39334, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-19T16:14:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732032898856, max timestamp=1732032898908Num uniques 859; 2024-11-19T16:14:58,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-19T16:14:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:58,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032958913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:58,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032958918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:58,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032958925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:58,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032958926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:58,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032958926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032959037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032959037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032959037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032959040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032959040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,177 INFO [master/32a8860bb9d6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-19T16:14:59,177 INFO [master/32a8860bb9d6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-19T16:14:59,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032959247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032959247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032959247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032959247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032959256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,282 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:59,323 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f11d1f13ea174a3cacb39b2aecfbc975_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f11d1f13ea174a3cacb39b2aecfbc975_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:59,324 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/0ec1bdbb22dc41aa804325c945d7e7ff, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:59,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/0ec1bdbb22dc41aa804325c945d7e7ff is 175, key is test_row_0/A:col10/1732032898853/Put/seqid=0 2024-11-19T16:14:59,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742015_1191 (size=65669) 2024-11-19T16:14:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T16:14:59,558 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:14:59,558 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:14:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-19T16:14:59,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032959557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T16:14:59,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032959557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032959557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032959557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,566 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:14:59,568 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:14:59,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:14:59,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:14:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032959569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:14:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:14:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:14:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T16:14:59,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T16:14:59,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:59,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:14:59,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:59,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:59,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:14:59,728 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/0ec1bdbb22dc41aa804325c945d7e7ff 2024-11-19T16:14:59,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/6721282c528c45bcb88ff7d01188d44c is 50, key is test_row_0/B:col10/1732032898853/Put/seqid=0 2024-11-19T16:14:59,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742016_1192 (size=9757) 2024-11-19T16:14:59,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/6721282c528c45bcb88ff7d01188d44c 2024-11-19T16:14:59,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/c8ac09e4ebec475d9e2131a99a45849c is 50, key is test_row_0/C:col10/1732032898853/Put/seqid=0 2024-11-19T16:14:59,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742017_1193 (size=9757) 2024-11-19T16:14:59,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/c8ac09e4ebec475d9e2131a99a45849c 2024-11-19T16:14:59,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/0ec1bdbb22dc41aa804325c945d7e7ff as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/0ec1bdbb22dc41aa804325c945d7e7ff 2024-11-19T16:14:59,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/0ec1bdbb22dc41aa804325c945d7e7ff, entries=350, sequenceid=229, filesize=64.1 K 2024-11-19T16:14:59,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/6721282c528c45bcb88ff7d01188d44c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/6721282c528c45bcb88ff7d01188d44c 2024-11-19T16:14:59,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/6721282c528c45bcb88ff7d01188d44c, entries=100, sequenceid=229, filesize=9.5 K 2024-11-19T16:14:59,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/c8ac09e4ebec475d9e2131a99a45849c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/c8ac09e4ebec475d9e2131a99a45849c 2024-11-19T16:14:59,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/c8ac09e4ebec475d9e2131a99a45849c, entries=100, sequenceid=229, filesize=9.5 K 2024-11-19T16:14:59,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=161.02 KB/164880 for 828823ece6f79c9635b39bcf387a8d6c in 941ms, sequenceid=229, compaction requested=true 2024-11-19T16:14:59,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:59,796 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:59,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:14:59,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:59,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:14:59,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:59,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:14:59,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T16:14:59,797 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128323 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:59,797 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/A is initiating minor compaction (all files) 2024-11-19T16:14:59,797 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/A in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:59,798 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/d10f5b653b95453ba64834e48416cd6c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/66359ac01aee4832b71ced0993a8c304, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/0ec1bdbb22dc41aa804325c945d7e7ff] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=125.3 K 2024-11-19T16:14:59,798 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:59,798 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/d10f5b653b95453ba64834e48416cd6c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/66359ac01aee4832b71ced0993a8c304, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/0ec1bdbb22dc41aa804325c945d7e7ff] 2024-11-19T16:14:59,798 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:59,798 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting d10f5b653b95453ba64834e48416cd6c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732032896645 2024-11-19T16:14:59,799 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66359ac01aee4832b71ced0993a8c304, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732032896703 2024-11-19T16:14:59,799 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:59,799 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/B is initiating minor compaction (all files) 2024-11-19T16:14:59,799 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/B in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:59,799 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/28ef2dcc408d46ecac3088175e91a891, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/c434ac7d856f42a184837f44be5c3e45, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/6721282c528c45bcb88ff7d01188d44c] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=33.7 K 2024-11-19T16:14:59,800 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ec1bdbb22dc41aa804325c945d7e7ff, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732032898822 2024-11-19T16:14:59,800 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 28ef2dcc408d46ecac3088175e91a891, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732032896645 2024-11-19T16:14:59,800 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting c434ac7d856f42a184837f44be5c3e45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732032896703 2024-11-19T16:14:59,801 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 6721282c528c45bcb88ff7d01188d44c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732032898847 2024-11-19T16:14:59,817 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:59,818 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#B#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:59,819 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/aa3d954d61314bf69e0cd132ee7a4076 is 50, key is test_row_0/B:col10/1732032898853/Put/seqid=0 2024-11-19T16:14:59,822 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111972e9566a757945d4aa957a2a9c86ee05_828823ece6f79c9635b39bcf387a8d6c store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:59,824 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111972e9566a757945d4aa957a2a9c86ee05_828823ece6f79c9635b39bcf387a8d6c, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:59,826 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111972e9566a757945d4aa957a2a9c86ee05_828823ece6f79c9635b39bcf387a8d6c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:59,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742018_1194 (size=12697) 2024-11-19T16:14:59,856 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/aa3d954d61314bf69e0cd132ee7a4076 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/aa3d954d61314bf69e0cd132ee7a4076 2024-11-19T16:14:59,862 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/B of 828823ece6f79c9635b39bcf387a8d6c into aa3d954d61314bf69e0cd132ee7a4076(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:59,862 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:59,862 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/B, priority=13, startTime=1732032899797; duration=0sec 2024-11-19T16:14:59,863 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:14:59,863 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:B 2024-11-19T16:14:59,863 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:14:59,864 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:14:59,865 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/C is initiating minor compaction (all files) 2024-11-19T16:14:59,865 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/C in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:59,865 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/43b8c5b061e64bd0a22f391f158fffc1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/383e7f1417d44006ae433747fbcf8630, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/c8ac09e4ebec475d9e2131a99a45849c] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=33.7 K 2024-11-19T16:14:59,865 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 43b8c5b061e64bd0a22f391f158fffc1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732032896645 2024-11-19T16:14:59,866 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 383e7f1417d44006ae433747fbcf8630, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732032896703 2024-11-19T16:14:59,866 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting c8ac09e4ebec475d9e2131a99a45849c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732032898847 2024-11-19T16:14:59,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742019_1195 (size=4469) 2024-11-19T16:14:59,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T16:14:59,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:14:59,876 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=161.02 KB heapSize=399.05 KB 2024-11-19T16:14:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:14:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:14:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:14:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:14:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T16:14:59,877 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#A#compaction#163 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:59,878 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/b833b43ca2524c639f256f6d2abe97e1 is 175, key is test_row_0/A:col10/1732032898853/Put/seqid=0 2024-11-19T16:14:59,880 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#C#compaction#164 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:14:59,881 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/4c6c7cd8d25f45a39a8ac2d4e0e0a3cb is 50, key is test_row_0/C:col10/1732032898853/Put/seqid=0 2024-11-19T16:14:59,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742020_1196 (size=31758) 2024-11-19T16:14:59,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c2baeba9f931423b854473c3f0ca4e1a_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032898924/Put/seqid=0 2024-11-19T16:14:59,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742022_1198 (size=12304) 2024-11-19T16:14:59,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:14:59,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742021_1197 (size=12697) 2024-11-19T16:14:59,940 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c2baeba9f931423b854473c3f0ca4e1a_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c2baeba9f931423b854473c3f0ca4e1a_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:14:59,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/33a071a0f06c429ebbcbe746aba7cbc6, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:14:59,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/33a071a0f06c429ebbcbe746aba7cbc6 is 175, key is test_row_0/A:col10/1732032898924/Put/seqid=0 2024-11-19T16:14:59,948 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/4c6c7cd8d25f45a39a8ac2d4e0e0a3cb as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/4c6c7cd8d25f45a39a8ac2d4e0e0a3cb 2024-11-19T16:14:59,955 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/C of 828823ece6f79c9635b39bcf387a8d6c into 4c6c7cd8d25f45a39a8ac2d4e0e0a3cb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:14:59,955 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:14:59,955 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/C, priority=13, startTime=1732032899797; duration=0sec 2024-11-19T16:14:59,955 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:14:59,955 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:C 2024-11-19T16:14:59,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742023_1199 (size=31105) 2024-11-19T16:14:59,959 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/33a071a0f06c429ebbcbe746aba7cbc6 2024-11-19T16:14:59,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/619754bcd7744b7d91d61de161afd538 is 50, key is test_row_0/B:col10/1732032898924/Put/seqid=0 2024-11-19T16:14:59,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742024_1200 (size=12151) 2024-11-19T16:14:59,982 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/619754bcd7744b7d91d61de161afd538 2024-11-19T16:14:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/fabb99695cb041c0b24724800c634566 is 50, key is test_row_0/C:col10/1732032898924/Put/seqid=0 2024-11-19T16:15:00,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742025_1201 (size=12151) 2024-11-19T16:15:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:00,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:00,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032960078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032960078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032960079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032960080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032960087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T16:15:00,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032960187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032960187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032960187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032960187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,304 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/b833b43ca2524c639f256f6d2abe97e1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/b833b43ca2524c639f256f6d2abe97e1 2024-11-19T16:15:00,312 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/A of 828823ece6f79c9635b39bcf387a8d6c into b833b43ca2524c639f256f6d2abe97e1(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:00,312 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:00,312 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/A, priority=13, startTime=1732032899796; duration=0sec 2024-11-19T16:15:00,312 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:00,312 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:A 2024-11-19T16:15:00,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032960396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032960396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032960406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032960406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,416 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/fabb99695cb041c0b24724800c634566 2024-11-19T16:15:00,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/33a071a0f06c429ebbcbe746aba7cbc6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/33a071a0f06c429ebbcbe746aba7cbc6 2024-11-19T16:15:00,425 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/33a071a0f06c429ebbcbe746aba7cbc6, entries=150, sequenceid=257, filesize=30.4 K 2024-11-19T16:15:00,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/619754bcd7744b7d91d61de161afd538 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/619754bcd7744b7d91d61de161afd538 2024-11-19T16:15:00,434 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/619754bcd7744b7d91d61de161afd538, entries=150, sequenceid=257, filesize=11.9 K 2024-11-19T16:15:00,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/fabb99695cb041c0b24724800c634566 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/fabb99695cb041c0b24724800c634566 2024-11-19T16:15:00,440 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/fabb99695cb041c0b24724800c634566, entries=150, sequenceid=257, filesize=11.9 K 2024-11-19T16:15:00,441 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~399 KB/408576, currentSize=40.25 KB/41220 for 828823ece6f79c9635b39bcf387a8d6c in 565ms, sequenceid=257, compaction requested=false 2024-11-19T16:15:00,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:00,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:00,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-19T16:15:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-19T16:15:00,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-19T16:15:00,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 875 msec 2024-11-19T16:15:00,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 892 msec 2024-11-19T16:15:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T16:15:00,697 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:00,697 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-19T16:15:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T16:15:00,699 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:00,699 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:00,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:00,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:15:00,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:15:00,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:00,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:15:00,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:00,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:15:00,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:00,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195cc36e50ff6b4227ab757b6c6a9f8e03_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032900707/Put/seqid=0 2024-11-19T16:15:00,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742026_1202 (size=14994) 2024-11-19T16:15:00,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032900727, max timestamp=1732032900736Num uniques 859; 2024-11-19T16:15:00,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032900727, max timestamp=1732032900736Num uniques 859; 2024-11-19T16:15:00,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:00,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:00,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032900727, max timestamp=1732032900736Num uniques 859; 2024-11-19T16:15:00,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:00,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032960754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032960758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032960760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032960761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T16:15:00,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T16:15:00,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:00,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:00,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:00,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:00,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:00,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032960866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032960867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032960877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032960877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T16:15:01,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:01,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T16:15:01,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032961077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:01,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032961077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032961087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:01,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032961087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032961096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:01,097 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 828823ece6f79c9635b39bcf387a8d6c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,122 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:01,127 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195cc36e50ff6b4227ab757b6c6a9f8e03_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195cc36e50ff6b4227ab757b6c6a9f8e03_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:01,128 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/42c1e489b197426088c3902684ea4dcd, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:01,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/42c1e489b197426088c3902684ea4dcd is 175, key is test_row_0/A:col10/1732032900707/Put/seqid=0 2024-11-19T16:15:01,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742027_1203 (size=39949) 2024-11-19T16:15:01,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T16:15:01,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:01,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T16:15:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T16:15:01,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032961387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:01,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032961387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032961397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:01,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032961397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T16:15:01,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,533 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/42c1e489b197426088c3902684ea4dcd 2024-11-19T16:15:01,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/f0ed56ab2b8841c8a11a3218beb0a9d1 is 50, key is test_row_0/B:col10/1732032900707/Put/seqid=0 2024-11-19T16:15:01,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742028_1204 (size=12301) 2024-11-19T16:15:01,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T16:15:01,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:01,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T16:15:01,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:01,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T16:15:01,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032961897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:01,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032961897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032961906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:01,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032961906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:01,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T16:15:01,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:01,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:01,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:01,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/f0ed56ab2b8841c8a11a3218beb0a9d1 2024-11-19T16:15:01,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/d88389efc0674dcf9f0e92b7c7238e46 is 50, key is test_row_0/C:col10/1732032900707/Put/seqid=0 2024-11-19T16:15:01,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742029_1205 (size=12301) 2024-11-19T16:15:02,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T16:15:02,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:02,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:02,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:02,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:02,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:02,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T16:15:02,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:02,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:02,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:02,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:02,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:02,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/d88389efc0674dcf9f0e92b7c7238e46 2024-11-19T16:15:02,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/42c1e489b197426088c3902684ea4dcd as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42c1e489b197426088c3902684ea4dcd 2024-11-19T16:15:02,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42c1e489b197426088c3902684ea4dcd, entries=200, sequenceid=269, filesize=39.0 K 2024-11-19T16:15:02,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/f0ed56ab2b8841c8a11a3218beb0a9d1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f0ed56ab2b8841c8a11a3218beb0a9d1 2024-11-19T16:15:02,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f0ed56ab2b8841c8a11a3218beb0a9d1, entries=150, sequenceid=269, filesize=12.0 K 2024-11-19T16:15:02,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/d88389efc0674dcf9f0e92b7c7238e46 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/d88389efc0674dcf9f0e92b7c7238e46 2024-11-19T16:15:02,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/d88389efc0674dcf9f0e92b7c7238e46, entries=150, sequenceid=269, filesize=12.0 K 2024-11-19T16:15:02,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 828823ece6f79c9635b39bcf387a8d6c in 1671ms, sequenceid=269, compaction requested=true 2024-11-19T16:15:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:02,379 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:02,379 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:02,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T16:15:02,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:02,380 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T16:15:02,380 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102812 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:02,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:15:02,380 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:02,381 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/B is initiating minor compaction (all files) 2024-11-19T16:15:02,381 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/A is initiating minor compaction (all files) 2024-11-19T16:15:02,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:02,381 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/B in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:02,381 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/A in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:02,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:15:02,381 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/aa3d954d61314bf69e0cd132ee7a4076, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/619754bcd7744b7d91d61de161afd538, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f0ed56ab2b8841c8a11a3218beb0a9d1] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=36.3 K 2024-11-19T16:15:02,381 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/b833b43ca2524c639f256f6d2abe97e1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/33a071a0f06c429ebbcbe746aba7cbc6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42c1e489b197426088c3902684ea4dcd] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=100.4 K 2024-11-19T16:15:02,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:02,381 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:02,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:15:02,381 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/b833b43ca2524c639f256f6d2abe97e1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/33a071a0f06c429ebbcbe746aba7cbc6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42c1e489b197426088c3902684ea4dcd] 2024-11-19T16:15:02,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:02,381 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting aa3d954d61314bf69e0cd132ee7a4076, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732032896713 2024-11-19T16:15:02,381 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting b833b43ca2524c639f256f6d2abe97e1, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732032896713 2024-11-19T16:15:02,382 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 619754bcd7744b7d91d61de161afd538, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732032898922 2024-11-19T16:15:02,382 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting f0ed56ab2b8841c8a11a3218beb0a9d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732032900075 2024-11-19T16:15:02,382 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33a071a0f06c429ebbcbe746aba7cbc6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732032898922 2024-11-19T16:15:02,382 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42c1e489b197426088c3902684ea4dcd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732032900075 2024-11-19T16:15:02,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111954ca1c18a4fd47eb93b86840847b258b_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032900759/Put/seqid=0 2024-11-19T16:15:02,394 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#B#compaction#172 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:02,395 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:02,395 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/9bb937ebf79f49bca8f723453ef84b64 is 50, key is test_row_0/B:col10/1732032900707/Put/seqid=0 2024-11-19T16:15:02,401 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119e46c5cdc45504812bd470fd65b4c9395_828823ece6f79c9635b39bcf387a8d6c store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:02,403 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119e46c5cdc45504812bd470fd65b4c9395_828823ece6f79c9635b39bcf387a8d6c, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:02,403 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e46c5cdc45504812bd470fd65b4c9395_828823ece6f79c9635b39bcf387a8d6c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:02,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742030_1206 (size=12454) 2024-11-19T16:15:02,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742031_1207 (size=12949) 2024-11-19T16:15:02,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:02,421 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111954ca1c18a4fd47eb93b86840847b258b_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111954ca1c18a4fd47eb93b86840847b258b_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:02,425 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/9bb937ebf79f49bca8f723453ef84b64 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/9bb937ebf79f49bca8f723453ef84b64 2024-11-19T16:15:02,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/5229f906cc0040b798a77bc7091c190e, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/5229f906cc0040b798a77bc7091c190e is 175, key is test_row_0/A:col10/1732032900759/Put/seqid=0 2024-11-19T16:15:02,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742032_1208 (size=4469) 2024-11-19T16:15:02,434 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#A#compaction#173 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:02,435 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/7771ebd6461f45c381cadd68adac2b98 is 175, key is test_row_0/A:col10/1732032900707/Put/seqid=0 2024-11-19T16:15:02,436 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/B of 828823ece6f79c9635b39bcf387a8d6c into 9bb937ebf79f49bca8f723453ef84b64(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:02,436 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:02,436 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/B, priority=13, startTime=1732032902379; duration=0sec 2024-11-19T16:15:02,437 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:02,437 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:B 2024-11-19T16:15:02,437 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:02,438 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:02,438 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/C is initiating minor compaction (all files) 2024-11-19T16:15:02,438 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/C in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:02,438 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/4c6c7cd8d25f45a39a8ac2d4e0e0a3cb, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/fabb99695cb041c0b24724800c634566, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/d88389efc0674dcf9f0e92b7c7238e46] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=36.3 K 2024-11-19T16:15:02,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742033_1209 (size=31255) 2024-11-19T16:15:02,439 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c6c7cd8d25f45a39a8ac2d4e0e0a3cb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732032896713 2024-11-19T16:15:02,440 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting fabb99695cb041c0b24724800c634566, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732032898922 2024-11-19T16:15:02,440 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting d88389efc0674dcf9f0e92b7c7238e46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732032900075 2024-11-19T16:15:02,451 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:02,452 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/30f666be542f42e19e0fab4aaed211f3 is 50, key is test_row_0/C:col10/1732032900707/Put/seqid=0 2024-11-19T16:15:02,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742034_1210 (size=31903) 2024-11-19T16:15:02,463 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/7771ebd6461f45c381cadd68adac2b98 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/7771ebd6461f45c381cadd68adac2b98 2024-11-19T16:15:02,469 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/A of 828823ece6f79c9635b39bcf387a8d6c into 7771ebd6461f45c381cadd68adac2b98(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:02,469 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:02,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742035_1211 (size=12949) 2024-11-19T16:15:02,469 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/A, priority=13, startTime=1732032902379; duration=0sec 2024-11-19T16:15:02,470 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:02,470 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:A 2024-11-19T16:15:02,474 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/30f666be542f42e19e0fab4aaed211f3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/30f666be542f42e19e0fab4aaed211f3 2024-11-19T16:15:02,479 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/C of 828823ece6f79c9635b39bcf387a8d6c into 30f666be542f42e19e0fab4aaed211f3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:02,479 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:02,479 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/C, priority=13, startTime=1732032902379; duration=0sec 2024-11-19T16:15:02,479 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:02,479 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:C 2024-11-19T16:15:02,840 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/5229f906cc0040b798a77bc7091c190e 2024-11-19T16:15:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T16:15:02,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/1b9f2c1d1e5346ddb4908e7c6959f7be is 50, key is test_row_0/B:col10/1732032900759/Put/seqid=0 2024-11-19T16:15:02,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742036_1212 (size=12301) 2024-11-19T16:15:02,852 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/1b9f2c1d1e5346ddb4908e7c6959f7be 2024-11-19T16:15:02,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/ad754a22e00644918929670704aa4a11 is 50, key is test_row_0/C:col10/1732032900759/Put/seqid=0 2024-11-19T16:15:02,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742037_1213 (size=12301) 2024-11-19T16:15:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:02,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:02,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032962918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:02,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032962919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032962920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032962927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:02,928 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 828823ece6f79c9635b39bcf387a8d6c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032963026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032963037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032963037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032963126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,129 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 828823ece6f79c9635b39bcf387a8d6c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032963236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032963247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032963247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,266 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/ad754a22e00644918929670704aa4a11 2024-11-19T16:15:03,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/5229f906cc0040b798a77bc7091c190e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/5229f906cc0040b798a77bc7091c190e 2024-11-19T16:15:03,276 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/5229f906cc0040b798a77bc7091c190e, entries=150, sequenceid=305, filesize=30.5 K 2024-11-19T16:15:03,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/1b9f2c1d1e5346ddb4908e7c6959f7be as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/1b9f2c1d1e5346ddb4908e7c6959f7be 2024-11-19T16:15:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,282 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/1b9f2c1d1e5346ddb4908e7c6959f7be, entries=150, sequenceid=305, filesize=12.0 K 2024-11-19T16:15:03,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/ad754a22e00644918929670704aa4a11 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/ad754a22e00644918929670704aa4a11 2024-11-19T16:15:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,288 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/ad754a22e00644918929670704aa4a11, entries=150, sequenceid=305, filesize=12.0 K 2024-11-19T16:15:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,289 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 828823ece6f79c9635b39bcf387a8d6c in 909ms, sequenceid=305, compaction requested=false 2024-11-19T16:15:03,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:03,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:03,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-19T16:15:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-19T16:15:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-19T16:15:03,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5920 sec 2024-11-19T16:15:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.5970 sec 2024-11-19T16:15:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:03,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T16:15:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:15:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:15:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:15:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119eae7c75f28604a198f475f386e47832d_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032902918/Put/seqid=0 2024-11-19T16:15:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742038_1214 (size=12454) 2024-11-19T16:15:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,583 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:03,587 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119eae7c75f28604a198f475f386e47832d_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119eae7c75f28604a198f475f386e47832d_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:03,588 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/55404492312f4ff8827d098a75d0ef26, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:03,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/55404492312f4ff8827d098a75d0ef26 is 175, key is test_row_0/A:col10/1732032902918/Put/seqid=0 2024-11-19T16:15:03,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742039_1215 (size=31255) 2024-11-19T16:15:03,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032963648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032903633, max timestamp=1732032903647Num uniques 859; 2024-11-19T16:15:03,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032903633, max timestamp=1732032903647Num uniques 859; 2024-11-19T16:15:03,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:15:03,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:15:03,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032903633, max timestamp=1732032903647Num uniques 859; 2024-11-19T16:15:03,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:15:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032963687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032963688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032963757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032963796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032963797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:03,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032963967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:03,998 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/55404492312f4ff8827d098a75d0ef26 2024-11-19T16:15:04,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/2708028a94a6463b8a1d1be9077dd9a2 is 50, key is test_row_0/B:col10/1732032902918/Put/seqid=0 2024-11-19T16:15:04,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032964006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:04,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032964007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:04,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742040_1216 (size=12301) 2024-11-19T16:15:04,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/2708028a94a6463b8a1d1be9077dd9a2 2024-11-19T16:15:04,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/7e7d784cf3b543fe9eb681569c9a2ad1 is 50, key is test_row_0/C:col10/1732032902918/Put/seqid=0 2024-11-19T16:15:04,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742041_1217 (size=12301) 2024-11-19T16:15:04,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032964277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:04,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032964317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:04,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032964317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:04,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/7e7d784cf3b543fe9eb681569c9a2ad1 2024-11-19T16:15:04,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/55404492312f4ff8827d098a75d0ef26 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/55404492312f4ff8827d098a75d0ef26 2024-11-19T16:15:04,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/55404492312f4ff8827d098a75d0ef26, entries=150, sequenceid=320, filesize=30.5 K 2024-11-19T16:15:04,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/2708028a94a6463b8a1d1be9077dd9a2 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/2708028a94a6463b8a1d1be9077dd9a2 2024-11-19T16:15:04,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/2708028a94a6463b8a1d1be9077dd9a2, entries=150, sequenceid=320, filesize=12.0 K 2024-11-19T16:15:04,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/7e7d784cf3b543fe9eb681569c9a2ad1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/7e7d784cf3b543fe9eb681569c9a2ad1 2024-11-19T16:15:04,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/7e7d784cf3b543fe9eb681569c9a2ad1, entries=150, sequenceid=320, filesize=12.0 K 2024-11-19T16:15:04,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 828823ece6f79c9635b39bcf387a8d6c in 900ms, sequenceid=320, compaction requested=true 2024-11-19T16:15:04,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:04,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:04,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:04,450 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:04,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:04,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:04,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:04,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T16:15:04,450 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:04,451 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:04,451 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/B is initiating minor compaction (all files) 2024-11-19T16:15:04,451 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/B in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:04,451 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/9bb937ebf79f49bca8f723453ef84b64, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/1b9f2c1d1e5346ddb4908e7c6959f7be, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/2708028a94a6463b8a1d1be9077dd9a2] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=36.7 K 2024-11-19T16:15:04,452 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:04,452 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/A is initiating minor compaction (all files) 2024-11-19T16:15:04,452 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/A in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:04,452 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/7771ebd6461f45c381cadd68adac2b98, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/5229f906cc0040b798a77bc7091c190e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/55404492312f4ff8827d098a75d0ef26] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=92.2 K 2024-11-19T16:15:04,452 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:04,452 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bb937ebf79f49bca8f723453ef84b64, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732032900075 2024-11-19T16:15:04,452 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b9f2c1d1e5346ddb4908e7c6959f7be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732032900752 2024-11-19T16:15:04,453 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/7771ebd6461f45c381cadd68adac2b98, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/5229f906cc0040b798a77bc7091c190e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/55404492312f4ff8827d098a75d0ef26] 2024-11-19T16:15:04,453 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 2708028a94a6463b8a1d1be9077dd9a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732032902917 2024-11-19T16:15:04,453 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7771ebd6461f45c381cadd68adac2b98, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732032900075 2024-11-19T16:15:04,454 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5229f906cc0040b798a77bc7091c190e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732032900752 2024-11-19T16:15:04,454 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55404492312f4ff8827d098a75d0ef26, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732032902917 2024-11-19T16:15:04,460 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:04,461 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#B#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:04,462 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/8491407a48b04493a321e46b387f3714 is 50, key is test_row_0/B:col10/1732032902918/Put/seqid=0 2024-11-19T16:15:04,464 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411196a651383c9644d25ae159ca7073e2952_828823ece6f79c9635b39bcf387a8d6c store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:04,466 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411196a651383c9644d25ae159ca7073e2952_828823ece6f79c9635b39bcf387a8d6c, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:04,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742042_1218 (size=13051) 2024-11-19T16:15:04,466 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196a651383c9644d25ae159ca7073e2952_828823ece6f79c9635b39bcf387a8d6c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:04,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742043_1219 (size=4469) 2024-11-19T16:15:04,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:04,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-19T16:15:04,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:15:04,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:04,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:15:04,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:04,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:15:04,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:04,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119232d5b1b92684b90b61b9ae566c81e0c_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032903686/Put/seqid=0 2024-11-19T16:15:04,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742044_1220 (size=14994) 2024-11-19T16:15:04,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:04,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032964827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:04,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032964832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:04,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:04,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032964836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:04,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T16:15:04,857 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:04,857 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-19T16:15:04,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T16:15:04,859 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:04,859 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:04,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:04,871 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#A#compaction#181 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:04,871 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/517164dbc0a54505a48b4c7cdbafce88 is 175, key is test_row_0/A:col10/1732032902918/Put/seqid=0 2024-11-19T16:15:04,872 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/8491407a48b04493a321e46b387f3714 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/8491407a48b04493a321e46b387f3714 2024-11-19T16:15:04,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742045_1221 (size=32005) 2024-11-19T16:15:04,878 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/B of 828823ece6f79c9635b39bcf387a8d6c into 8491407a48b04493a321e46b387f3714(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:04,878 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:04,878 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/B, priority=13, startTime=1732032904450; duration=0sec 2024-11-19T16:15:04,878 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:04,878 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:B 2024-11-19T16:15:04,878 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:04,879 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:04,879 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/C is initiating minor compaction (all files) 2024-11-19T16:15:04,879 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/C in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:04,879 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/30f666be542f42e19e0fab4aaed211f3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/ad754a22e00644918929670704aa4a11, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/7e7d784cf3b543fe9eb681569c9a2ad1] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=36.7 K 2024-11-19T16:15:04,879 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 30f666be542f42e19e0fab4aaed211f3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732032900075 2024-11-19T16:15:04,880 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting ad754a22e00644918929670704aa4a11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732032900752 2024-11-19T16:15:04,880 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e7d784cf3b543fe9eb681569c9a2ad1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732032902917 2024-11-19T16:15:04,888 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#C#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:04,888 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/99ab75ff5e9b4f78a6734924f2329501 is 50, key is test_row_0/C:col10/1732032902918/Put/seqid=0 2024-11-19T16:15:04,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742046_1222 (size=13051) 2024-11-19T16:15:04,897 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/99ab75ff5e9b4f78a6734924f2329501 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/99ab75ff5e9b4f78a6734924f2329501 2024-11-19T16:15:04,902 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/C of 828823ece6f79c9635b39bcf387a8d6c into 99ab75ff5e9b4f78a6734924f2329501(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:04,902 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:04,902 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/C, priority=13, startTime=1732032904450; duration=0sec 2024-11-19T16:15:04,902 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:04,902 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:C 2024-11-19T16:15:04,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:04,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54806 deadline: 1732032964946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:04,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032964947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:04,953 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 828823ece6f79c9635b39bcf387a8d6c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T16:15:05,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T16:15:05,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:05,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032965156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:05,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:05,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T16:15:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T16:15:05,203 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:05,206 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119232d5b1b92684b90b61b9ae566c81e0c_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119232d5b1b92684b90b61b9ae566c81e0c_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:05,207 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/b956dd982f814ba692341bd87987506f, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:05,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/b956dd982f814ba692341bd87987506f is 175, key is test_row_0/A:col10/1732032903686/Put/seqid=0 2024-11-19T16:15:05,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742047_1223 (size=39949) 2024-11-19T16:15:05,281 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/517164dbc0a54505a48b4c7cdbafce88 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/517164dbc0a54505a48b4c7cdbafce88 2024-11-19T16:15:05,286 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/A of 828823ece6f79c9635b39bcf387a8d6c into 517164dbc0a54505a48b4c7cdbafce88(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:05,287 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:05,287 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/A, priority=13, startTime=1732032904450; duration=0sec 2024-11-19T16:15:05,287 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:05,287 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:A 2024-11-19T16:15:05,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T16:15:05,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:05,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:05,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T16:15:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032965467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:05,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:05,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:05,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:05,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T16:15:05,614 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/b956dd982f814ba692341bd87987506f 2024-11-19T16:15:05,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T16:15:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/db437c31ade1451e9838d18b553d3fac is 50, key is test_row_0/B:col10/1732032903686/Put/seqid=0 2024-11-19T16:15:05,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742048_1224 (size=12301) 2024-11-19T16:15:05,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T16:15:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032965847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:05,849 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 828823ece6f79c9635b39bcf387a8d6c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:05,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032965856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:05,859 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 828823ece6f79c9635b39bcf387a8d6c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:05,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T16:15:05,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:05,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:05,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:05,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032965977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:05,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:05,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:05,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T16:15:06,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/db437c31ade1451e9838d18b553d3fac 2024-11-19T16:15:06,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/ec1053c861944b238c4e2ef48527e1d9 is 50, key is test_row_0/C:col10/1732032903686/Put/seqid=0 2024-11-19T16:15:06,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742049_1225 (size=12301) 2024-11-19T16:15:06,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/ec1053c861944b238c4e2ef48527e1d9 2024-11-19T16:15:06,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/b956dd982f814ba692341bd87987506f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/b956dd982f814ba692341bd87987506f 2024-11-19T16:15:06,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/b956dd982f814ba692341bd87987506f, entries=200, sequenceid=354, filesize=39.0 K 2024-11-19T16:15:06,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/db437c31ade1451e9838d18b553d3fac as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/db437c31ade1451e9838d18b553d3fac 2024-11-19T16:15:06,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/db437c31ade1451e9838d18b553d3fac, entries=150, sequenceid=354, filesize=12.0 K 2024-11-19T16:15:06,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/ec1053c861944b238c4e2ef48527e1d9 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/ec1053c861944b238c4e2ef48527e1d9 2024-11-19T16:15:06,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T16:15:06,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:06,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:06,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:06,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:06,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:06,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/ec1053c861944b238c4e2ef48527e1d9, entries=150, sequenceid=354, filesize=12.0 K 2024-11-19T16:15:06,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 828823ece6f79c9635b39bcf387a8d6c in 1296ms, sequenceid=354, compaction requested=false 2024-11-19T16:15:06,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:06,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T16:15:06,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:06,233 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T16:15:06,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:15:06,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:06,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:15:06,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:06,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:15:06,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:06,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194c834bf0d7194ebc8a447a0b50be9a36_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032904817/Put/seqid=0 2024-11-19T16:15:06,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742050_1226 (size=12454) 2024-11-19T16:15:06,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:06,648 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194c834bf0d7194ebc8a447a0b50be9a36_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194c834bf0d7194ebc8a447a0b50be9a36_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/1de65b8ddb4346d5a711515f615a2a40, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/1de65b8ddb4346d5a711515f615a2a40 is 175, key is test_row_0/A:col10/1732032904817/Put/seqid=0 2024-11-19T16:15:06,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742051_1227 (size=31255) 2024-11-19T16:15:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:06,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. as already flushing 2024-11-19T16:15:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T16:15:07,056 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/1de65b8ddb4346d5a711515f615a2a40 2024-11-19T16:15:07,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032907025, max timestamp=1732032907062Num uniques 859; 2024-11-19T16:15:07,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032907025, max timestamp=1732032907062Num uniques 859; 2024-11-19T16:15:07,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032907025, max timestamp=1732032907062Num uniques 859; 2024-11-19T16:15:07,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:07,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:07,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:07,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/436c04daa4ea443286b2f7b9b889341e is 50, key is test_row_0/B:col10/1732032904817/Put/seqid=0 2024-11-19T16:15:07,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742052_1228 (size=12301) 2024-11-19T16:15:07,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032967114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:07,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:07,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54836 deadline: 1732032967177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:07,181 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 828823ece6f79c9635b39bcf387a8d6c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:07,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032967226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:07,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:07,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032967436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:07,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:07,478 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/436c04daa4ea443286b2f7b9b889341e 2024-11-19T16:15:07,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/461b3c79e40b444f852b67ddf2d18a8c is 50, key is test_row_0/C:col10/1732032904817/Put/seqid=0 2024-11-19T16:15:07,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742053_1229 (size=12301) 2024-11-19T16:15:07,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54814 deadline: 1732032967746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:07,790 INFO [Thread-655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-655. 2024-11-19T16:15:07,790 DEBUG [Thread-655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:07,790 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:07,790 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:07,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:07,792 INFO [Thread-661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-661. 2024-11-19T16:15:07,792 DEBUG [Thread-661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:07,792 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:07,792 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:07,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:07,795 INFO [Thread-657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-657. 2024-11-19T16:15:07,795 DEBUG [Thread-657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:07,795 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:07,795 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:07,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:07,798 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-11-19T16:15:07,798 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:07,798 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:07,798 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:07,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:07,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1732032967867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:07,867 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 828823ece6f79c9635b39bcf387a8d6c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:07,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:07,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54784 deadline: 1732032967887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:07,888 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 828823ece6f79c9635b39bcf387a8d6c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:07,894 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/461b3c79e40b444f852b67ddf2d18a8c 2024-11-19T16:15:07,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/1de65b8ddb4346d5a711515f615a2a40 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/1de65b8ddb4346d5a711515f615a2a40 2024-11-19T16:15:07,902 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/1de65b8ddb4346d5a711515f615a2a40, entries=150, sequenceid=368, filesize=30.5 K 2024-11-19T16:15:07,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/436c04daa4ea443286b2f7b9b889341e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/436c04daa4ea443286b2f7b9b889341e 2024-11-19T16:15:07,907 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/436c04daa4ea443286b2f7b9b889341e, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T16:15:07,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/461b3c79e40b444f852b67ddf2d18a8c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/461b3c79e40b444f852b67ddf2d18a8c 2024-11-19T16:15:07,912 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/461b3c79e40b444f852b67ddf2d18a8c, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T16:15:07,913 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 828823ece6f79c9635b39bcf387a8d6c in 1680ms, sequenceid=368, compaction requested=true 2024-11-19T16:15:07,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:07,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:07,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-19T16:15:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-19T16:15:07,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-19T16:15:07,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0550 sec 2024-11-19T16:15:07,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 3.0590 sec 2024-11-19T16:15:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:08,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T16:15:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:15:08,259 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-19T16:15:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:15:08,259 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:08,259 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:15:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:08,259 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:08,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:08,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e2d92001a5e2422a9804e7842616438a_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_0/A:col10/1732032907113/Put/seqid=0 2024-11-19T16:15:08,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742054_1230 (size=12454) 2024-11-19T16:15:08,673 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:08,682 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e2d92001a5e2422a9804e7842616438a_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e2d92001a5e2422a9804e7842616438a_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:08,683 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/3aad83e5c4a24efc8a0892652b043e7c, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:08,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/3aad83e5c4a24efc8a0892652b043e7c is 175, key is test_row_0/A:col10/1732032907113/Put/seqid=0 2024-11-19T16:15:08,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742055_1231 (size=31255) 2024-11-19T16:15:09,001 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-19T16:15:09,001 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:09,001 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:09,001 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:09,001 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T16:15:09,038 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:09,091 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/3aad83e5c4a24efc8a0892652b043e7c 2024-11-19T16:15:09,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/f6bdfd49042c4cc28094632eea440cd4 is 50, key is test_row_0/B:col10/1732032907113/Put/seqid=0 2024-11-19T16:15:09,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742056_1232 (size=12301) 2024-11-19T16:15:09,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/f6bdfd49042c4cc28094632eea440cd4 2024-11-19T16:15:09,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/a18e766f0f65400fb2d138f34d1fba6c is 50, key is test_row_0/C:col10/1732032907113/Put/seqid=0 2024-11-19T16:15:09,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742057_1233 (size=12301) 2024-11-19T16:15:09,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/a18e766f0f65400fb2d138f34d1fba6c 2024-11-19T16:15:09,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/3aad83e5c4a24efc8a0892652b043e7c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/3aad83e5c4a24efc8a0892652b043e7c 2024-11-19T16:15:09,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/3aad83e5c4a24efc8a0892652b043e7c, entries=150, sequenceid=404, filesize=30.5 K 2024-11-19T16:15:09,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/f6bdfd49042c4cc28094632eea440cd4 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f6bdfd49042c4cc28094632eea440cd4 2024-11-19T16:15:09,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f6bdfd49042c4cc28094632eea440cd4, entries=150, sequenceid=404, filesize=12.0 K 2024-11-19T16:15:09,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/a18e766f0f65400fb2d138f34d1fba6c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a18e766f0f65400fb2d138f34d1fba6c 2024-11-19T16:15:09,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a18e766f0f65400fb2d138f34d1fba6c, entries=150, sequenceid=404, filesize=12.0 K 2024-11-19T16:15:09,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=6.71 KB/6870 for 828823ece6f79c9635b39bcf387a8d6c in 1738ms, sequenceid=404, compaction requested=true 2024-11-19T16:15:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:09,998 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 828823ece6f79c9635b39bcf387a8d6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:09,998 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:09,999 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:09,999 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:09,999 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/B is initiating minor compaction (all files) 2024-11-19T16:15:09,999 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/A is initiating minor compaction (all files) 2024-11-19T16:15:09,999 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/A in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:09,999 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/B in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:09,999 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/8491407a48b04493a321e46b387f3714, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/db437c31ade1451e9838d18b553d3fac, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/436c04daa4ea443286b2f7b9b889341e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f6bdfd49042c4cc28094632eea440cd4] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=48.8 K 2024-11-19T16:15:09,999 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/517164dbc0a54505a48b4c7cdbafce88, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/b956dd982f814ba692341bd87987506f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/1de65b8ddb4346d5a711515f615a2a40, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/3aad83e5c4a24efc8a0892652b043e7c] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=131.3 K 2024-11-19T16:15:10,000 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:10,000 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/517164dbc0a54505a48b4c7cdbafce88, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/b956dd982f814ba692341bd87987506f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/1de65b8ddb4346d5a711515f615a2a40, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/3aad83e5c4a24efc8a0892652b043e7c] 2024-11-19T16:15:10,000 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 8491407a48b04493a321e46b387f3714, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732032902917 2024-11-19T16:15:10,000 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 517164dbc0a54505a48b4c7cdbafce88, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732032902917 2024-11-19T16:15:10,000 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting db437c31ade1451e9838d18b553d3fac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732032903664 2024-11-19T16:15:10,000 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting b956dd982f814ba692341bd87987506f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732032903664 2024-11-19T16:15:10,000 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 436c04daa4ea443286b2f7b9b889341e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732032904817 2024-11-19T16:15:10,001 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1de65b8ddb4346d5a711515f615a2a40, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732032904817 2024-11-19T16:15:10,001 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting f6bdfd49042c4cc28094632eea440cd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732032907103 2024-11-19T16:15:10,001 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aad83e5c4a24efc8a0892652b043e7c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732032907103 2024-11-19T16:15:10,011 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:10,013 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#B#compaction#192 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:10,014 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119b83b7f5eafb34a1a97f0df54d6714829_828823ece6f79c9635b39bcf387a8d6c store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:10,014 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/92dfa32f0de745f183ab7e473d035e53 is 50, key is test_row_0/B:col10/1732032907113/Put/seqid=0 2024-11-19T16:15:10,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742058_1234 (size=13187) 2024-11-19T16:15:10,045 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119b83b7f5eafb34a1a97f0df54d6714829_828823ece6f79c9635b39bcf387a8d6c, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:10,045 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b83b7f5eafb34a1a97f0df54d6714829_828823ece6f79c9635b39bcf387a8d6c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:10,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742059_1235 (size=4469) 2024-11-19T16:15:10,423 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/92dfa32f0de745f183ab7e473d035e53 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/92dfa32f0de745f183ab7e473d035e53 2024-11-19T16:15:10,429 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/B of 828823ece6f79c9635b39bcf387a8d6c into 92dfa32f0de745f183ab7e473d035e53(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:10,429 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:10,429 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/B, priority=12, startTime=1732032909998; duration=0sec 2024-11-19T16:15:10,429 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:10,429 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:B 2024-11-19T16:15:10,429 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:10,431 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:10,431 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 828823ece6f79c9635b39bcf387a8d6c/C is initiating minor compaction (all files) 2024-11-19T16:15:10,431 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 828823ece6f79c9635b39bcf387a8d6c/C in TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:10,431 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/99ab75ff5e9b4f78a6734924f2329501, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/ec1053c861944b238c4e2ef48527e1d9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/461b3c79e40b444f852b67ddf2d18a8c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a18e766f0f65400fb2d138f34d1fba6c] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp, totalSize=48.8 K 2024-11-19T16:15:10,431 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 99ab75ff5e9b4f78a6734924f2329501, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732032902917 2024-11-19T16:15:10,432 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting ec1053c861944b238c4e2ef48527e1d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732032903664 2024-11-19T16:15:10,432 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 461b3c79e40b444f852b67ddf2d18a8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732032904817 2024-11-19T16:15:10,432 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a18e766f0f65400fb2d138f34d1fba6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732032907103 2024-11-19T16:15:10,444 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#C#compaction#194 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:10,444 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/2025b63a344345b58f2ab2cd60510c3c is 50, key is test_row_0/C:col10/1732032907113/Put/seqid=0 2024-11-19T16:15:10,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742060_1236 (size=13187) 2024-11-19T16:15:10,451 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 828823ece6f79c9635b39bcf387a8d6c#A#compaction#193 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:10,451 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/bba8680a33cf42ab9bc4a2c9eef64179 is 175, key is test_row_0/A:col10/1732032907113/Put/seqid=0 2024-11-19T16:15:10,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742061_1237 (size=32141) 2024-11-19T16:15:10,859 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/2025b63a344345b58f2ab2cd60510c3c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/2025b63a344345b58f2ab2cd60510c3c 2024-11-19T16:15:10,863 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/bba8680a33cf42ab9bc4a2c9eef64179 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/bba8680a33cf42ab9bc4a2c9eef64179 2024-11-19T16:15:10,864 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/C of 828823ece6f79c9635b39bcf387a8d6c into 2025b63a344345b58f2ab2cd60510c3c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:10,864 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:10,864 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/C, priority=12, startTime=1732032909998; duration=0sec 2024-11-19T16:15:10,864 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:10,864 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:C 2024-11-19T16:15:10,867 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 828823ece6f79c9635b39bcf387a8d6c/A of 828823ece6f79c9635b39bcf387a8d6c into bba8680a33cf42ab9bc4a2c9eef64179(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:10,867 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 828823ece6f79c9635b39bcf387a8d6c: 2024-11-19T16:15:10,867 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., storeName=828823ece6f79c9635b39bcf387a8d6c/A, priority=12, startTime=1732032909998; duration=0sec 2024-11-19T16:15:10,868 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:10,868 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 828823ece6f79c9635b39bcf387a8d6c:A 2024-11-19T16:15:11,912 INFO [Thread-644 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-644. 2024-11-19T16:15:11,912 DEBUG [Thread-644 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:11,912 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:11,912 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:11,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:11,931 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-19T16:15:11,931 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:11,931 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:11,931 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:11,931 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:12,300 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T16:15:17,208 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-19T16:15:17,208 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:17,208 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:17,208 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:17,208 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3218 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2989 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1308 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3923 rows 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1306 2024-11-19T16:15:17,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3915 rows 2024-11-19T16:15:17,209 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T16:15:17,209 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T16:15:17,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:17,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:17,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:17,210 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-19T16:15:17,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T16:15:17,215 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032917215"}]},"ts":"1732032917215"} 2024-11-19T16:15:17,218 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T16:15:17,218 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T16:15:17,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T16:15:17,221 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, UNASSIGN}] 2024-11-19T16:15:17,221 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, UNASSIGN 2024-11-19T16:15:17,222 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=828823ece6f79c9635b39bcf387a8d6c, regionState=CLOSING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:17,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, UNASSIGN because future has completed 2024-11-19T16:15:17,223 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T16:15:17,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:15:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T16:15:17,376 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:17,376 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T16:15:17,376 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 828823ece6f79c9635b39bcf387a8d6c, disabling compactions & flushes 2024-11-19T16:15:17,376 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:17,376 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:17,377 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. after waiting 0 ms 2024-11-19T16:15:17,377 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:17,377 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 828823ece6f79c9635b39bcf387a8d6c 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T16:15:17,377 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=A 2024-11-19T16:15:17,377 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:17,377 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=B 2024-11-19T16:15:17,377 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:17,377 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 828823ece6f79c9635b39bcf387a8d6c, store=C 2024-11-19T16:15:17,377 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:17,387 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111908079d55f5604a00b925d85bb34d389f_828823ece6f79c9635b39bcf387a8d6c is 50, key is test_row_1/A:col10/1732032911908/Put/seqid=0 2024-11-19T16:15:17,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742062_1238 (size=9914) 2024-11-19T16:15:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T16:15:17,794 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:17,807 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111908079d55f5604a00b925d85bb34d389f_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111908079d55f5604a00b925d85bb34d389f_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:17,808 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/585eb8cc81ed4e9f871cd3614db52869, store: [table=TestAcidGuarantees family=A region=828823ece6f79c9635b39bcf387a8d6c] 2024-11-19T16:15:17,809 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/585eb8cc81ed4e9f871cd3614db52869 is 175, key is test_row_1/A:col10/1732032911908/Put/seqid=0 2024-11-19T16:15:17,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742063_1239 (size=22561) 2024-11-19T16:15:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T16:15:18,214 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/585eb8cc81ed4e9f871cd3614db52869 2024-11-19T16:15:18,232 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/b90f3ad0ad6c4ac984cd1e74ca35e2e2 is 50, key is test_row_1/B:col10/1732032911908/Put/seqid=0 2024-11-19T16:15:18,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742064_1240 (size=9857) 2024-11-19T16:15:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T16:15:18,650 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/b90f3ad0ad6c4ac984cd1e74ca35e2e2 2024-11-19T16:15:18,658 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/7df4760cb8084f79b82fc786fd2aca99 is 50, key is test_row_1/C:col10/1732032911908/Put/seqid=0 2024-11-19T16:15:18,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742065_1241 (size=9857) 2024-11-19T16:15:19,067 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/7df4760cb8084f79b82fc786fd2aca99 2024-11-19T16:15:19,072 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/A/585eb8cc81ed4e9f871cd3614db52869 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/585eb8cc81ed4e9f871cd3614db52869 2024-11-19T16:15:19,079 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/585eb8cc81ed4e9f871cd3614db52869, entries=100, sequenceid=414, filesize=22.0 K 2024-11-19T16:15:19,080 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/B/b90f3ad0ad6c4ac984cd1e74ca35e2e2 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/b90f3ad0ad6c4ac984cd1e74ca35e2e2 2024-11-19T16:15:19,084 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/b90f3ad0ad6c4ac984cd1e74ca35e2e2, entries=100, sequenceid=414, filesize=9.6 K 2024-11-19T16:15:19,084 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/.tmp/C/7df4760cb8084f79b82fc786fd2aca99 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/7df4760cb8084f79b82fc786fd2aca99 2024-11-19T16:15:19,088 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/7df4760cb8084f79b82fc786fd2aca99, entries=100, sequenceid=414, filesize=9.6 K 2024-11-19T16:15:19,089 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 828823ece6f79c9635b39bcf387a8d6c in 1712ms, sequenceid=414, compaction requested=false 2024-11-19T16:15:19,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42b75fa0ecae4a3f8c633d428fde5e6e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/06cf6b9f5aac43e09e7154b62bad6da9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/c0119c6b1cb1424eb36b61fd491269ec, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/6a8a538b58c5403b8cb21c29cfa778d3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/880dcc4350ae4e5c8f8b5636428a03f9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/d275d6b0c62d465795cfedf6fea4f603, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/fd08ef51c0da4d5db9de4c2a586164a8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/e8112b06b22440aebce1d80cd9ed2591, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/692f7f5a4eaa473f890ff43ba7b79e4f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/ec0424dc21864c21bd454d2446228a3a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/58878f6d3f634eb98b99d300a3c05e74, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/c3103081ba694924b48f7ad7bcd3d587, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/2750b1bdeb4b4f43b6ee614f73154085, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/d10f5b653b95453ba64834e48416cd6c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/66359ac01aee4832b71ced0993a8c304, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/0ec1bdbb22dc41aa804325c945d7e7ff, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/b833b43ca2524c639f256f6d2abe97e1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/33a071a0f06c429ebbcbe746aba7cbc6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42c1e489b197426088c3902684ea4dcd, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/7771ebd6461f45c381cadd68adac2b98, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/5229f906cc0040b798a77bc7091c190e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/517164dbc0a54505a48b4c7cdbafce88, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/55404492312f4ff8827d098a75d0ef26, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/b956dd982f814ba692341bd87987506f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/1de65b8ddb4346d5a711515f615a2a40, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/3aad83e5c4a24efc8a0892652b043e7c] to archive 2024-11-19T16:15:19,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:15:19,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42b75fa0ecae4a3f8c633d428fde5e6e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42b75fa0ecae4a3f8c633d428fde5e6e 2024-11-19T16:15:19,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/06cf6b9f5aac43e09e7154b62bad6da9 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/06cf6b9f5aac43e09e7154b62bad6da9 2024-11-19T16:15:19,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/c0119c6b1cb1424eb36b61fd491269ec to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/c0119c6b1cb1424eb36b61fd491269ec 2024-11-19T16:15:19,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/6a8a538b58c5403b8cb21c29cfa778d3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/6a8a538b58c5403b8cb21c29cfa778d3 2024-11-19T16:15:19,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/880dcc4350ae4e5c8f8b5636428a03f9 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/880dcc4350ae4e5c8f8b5636428a03f9 2024-11-19T16:15:19,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/d275d6b0c62d465795cfedf6fea4f603 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/d275d6b0c62d465795cfedf6fea4f603 2024-11-19T16:15:19,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/fd08ef51c0da4d5db9de4c2a586164a8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/fd08ef51c0da4d5db9de4c2a586164a8 2024-11-19T16:15:19,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/e8112b06b22440aebce1d80cd9ed2591 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/e8112b06b22440aebce1d80cd9ed2591 2024-11-19T16:15:19,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/692f7f5a4eaa473f890ff43ba7b79e4f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/692f7f5a4eaa473f890ff43ba7b79e4f 2024-11-19T16:15:19,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/ec0424dc21864c21bd454d2446228a3a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/ec0424dc21864c21bd454d2446228a3a 2024-11-19T16:15:19,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/58878f6d3f634eb98b99d300a3c05e74 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/58878f6d3f634eb98b99d300a3c05e74 2024-11-19T16:15:19,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/c3103081ba694924b48f7ad7bcd3d587 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/c3103081ba694924b48f7ad7bcd3d587 2024-11-19T16:15:19,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/2750b1bdeb4b4f43b6ee614f73154085 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/2750b1bdeb4b4f43b6ee614f73154085 2024-11-19T16:15:19,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/d10f5b653b95453ba64834e48416cd6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/d10f5b653b95453ba64834e48416cd6c 2024-11-19T16:15:19,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/66359ac01aee4832b71ced0993a8c304 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/66359ac01aee4832b71ced0993a8c304 2024-11-19T16:15:19,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/0ec1bdbb22dc41aa804325c945d7e7ff to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/0ec1bdbb22dc41aa804325c945d7e7ff 2024-11-19T16:15:19,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/b833b43ca2524c639f256f6d2abe97e1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/b833b43ca2524c639f256f6d2abe97e1 2024-11-19T16:15:19,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/33a071a0f06c429ebbcbe746aba7cbc6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/33a071a0f06c429ebbcbe746aba7cbc6 2024-11-19T16:15:19,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42c1e489b197426088c3902684ea4dcd to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/42c1e489b197426088c3902684ea4dcd 2024-11-19T16:15:19,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/7771ebd6461f45c381cadd68adac2b98 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/7771ebd6461f45c381cadd68adac2b98 2024-11-19T16:15:19,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/5229f906cc0040b798a77bc7091c190e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/5229f906cc0040b798a77bc7091c190e 2024-11-19T16:15:19,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/517164dbc0a54505a48b4c7cdbafce88 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/517164dbc0a54505a48b4c7cdbafce88 2024-11-19T16:15:19,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/55404492312f4ff8827d098a75d0ef26 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/55404492312f4ff8827d098a75d0ef26 2024-11-19T16:15:19,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/b956dd982f814ba692341bd87987506f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/b956dd982f814ba692341bd87987506f 2024-11-19T16:15:19,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/1de65b8ddb4346d5a711515f615a2a40 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/1de65b8ddb4346d5a711515f615a2a40 2024-11-19T16:15:19,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/3aad83e5c4a24efc8a0892652b043e7c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/3aad83e5c4a24efc8a0892652b043e7c 2024-11-19T16:15:19,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/2b060cbea3f141ffb52499544d6ca75b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/65172e8e819e47c1b2d612e413f66ea3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/67bf2feb4c404ff390b170af9b946073, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/511c3ae536914114b3917c0ee8ad031a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/63d23a51ab1c42cfb327ead0ee41bd9f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/6ca58ada417c4e419110a016f8ded5f1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/e7afecfafb4b4c728ffa551351e0703e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/e776e866585b4b1ca8dcd28e79ff21ca, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/1dd568beff7347e99d34265d6b8c8941, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f5e124906efd47e58394725dad20b0f1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/792b374e18e646efb8ac8de63a24f6f4, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/41a6f0fe506d40529d11d2a6d3f7441a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/28ef2dcc408d46ecac3088175e91a891, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/d0743313b39449b7a21da845a544ed61, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/c434ac7d856f42a184837f44be5c3e45, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/aa3d954d61314bf69e0cd132ee7a4076, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/6721282c528c45bcb88ff7d01188d44c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/619754bcd7744b7d91d61de161afd538, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/9bb937ebf79f49bca8f723453ef84b64, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f0ed56ab2b8841c8a11a3218beb0a9d1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/1b9f2c1d1e5346ddb4908e7c6959f7be, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/8491407a48b04493a321e46b387f3714, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/2708028a94a6463b8a1d1be9077dd9a2, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/db437c31ade1451e9838d18b553d3fac, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/436c04daa4ea443286b2f7b9b889341e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f6bdfd49042c4cc28094632eea440cd4] to archive 2024-11-19T16:15:19,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:15:19,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/2b060cbea3f141ffb52499544d6ca75b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/2b060cbea3f141ffb52499544d6ca75b 2024-11-19T16:15:19,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/65172e8e819e47c1b2d612e413f66ea3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/65172e8e819e47c1b2d612e413f66ea3 2024-11-19T16:15:19,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/67bf2feb4c404ff390b170af9b946073 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/67bf2feb4c404ff390b170af9b946073 2024-11-19T16:15:19,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/511c3ae536914114b3917c0ee8ad031a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/511c3ae536914114b3917c0ee8ad031a 2024-11-19T16:15:19,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/63d23a51ab1c42cfb327ead0ee41bd9f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/63d23a51ab1c42cfb327ead0ee41bd9f 2024-11-19T16:15:19,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/6ca58ada417c4e419110a016f8ded5f1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/6ca58ada417c4e419110a016f8ded5f1 2024-11-19T16:15:19,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/e7afecfafb4b4c728ffa551351e0703e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/e7afecfafb4b4c728ffa551351e0703e 2024-11-19T16:15:19,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/e776e866585b4b1ca8dcd28e79ff21ca to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/e776e866585b4b1ca8dcd28e79ff21ca 2024-11-19T16:15:19,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/1dd568beff7347e99d34265d6b8c8941 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/1dd568beff7347e99d34265d6b8c8941 2024-11-19T16:15:19,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f5e124906efd47e58394725dad20b0f1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f5e124906efd47e58394725dad20b0f1 2024-11-19T16:15:19,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/792b374e18e646efb8ac8de63a24f6f4 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/792b374e18e646efb8ac8de63a24f6f4 2024-11-19T16:15:19,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/41a6f0fe506d40529d11d2a6d3f7441a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/41a6f0fe506d40529d11d2a6d3f7441a 2024-11-19T16:15:19,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/28ef2dcc408d46ecac3088175e91a891 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/28ef2dcc408d46ecac3088175e91a891 2024-11-19T16:15:19,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/d0743313b39449b7a21da845a544ed61 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/d0743313b39449b7a21da845a544ed61 2024-11-19T16:15:19,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/c434ac7d856f42a184837f44be5c3e45 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/c434ac7d856f42a184837f44be5c3e45 2024-11-19T16:15:19,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/aa3d954d61314bf69e0cd132ee7a4076 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/aa3d954d61314bf69e0cd132ee7a4076 2024-11-19T16:15:19,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/6721282c528c45bcb88ff7d01188d44c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/6721282c528c45bcb88ff7d01188d44c 2024-11-19T16:15:19,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/619754bcd7744b7d91d61de161afd538 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/619754bcd7744b7d91d61de161afd538 2024-11-19T16:15:19,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/9bb937ebf79f49bca8f723453ef84b64 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/9bb937ebf79f49bca8f723453ef84b64 2024-11-19T16:15:19,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f0ed56ab2b8841c8a11a3218beb0a9d1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f0ed56ab2b8841c8a11a3218beb0a9d1 2024-11-19T16:15:19,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/1b9f2c1d1e5346ddb4908e7c6959f7be to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/1b9f2c1d1e5346ddb4908e7c6959f7be 2024-11-19T16:15:19,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/8491407a48b04493a321e46b387f3714 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/8491407a48b04493a321e46b387f3714 2024-11-19T16:15:19,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/2708028a94a6463b8a1d1be9077dd9a2 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/2708028a94a6463b8a1d1be9077dd9a2 2024-11-19T16:15:19,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/db437c31ade1451e9838d18b553d3fac to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/db437c31ade1451e9838d18b553d3fac 2024-11-19T16:15:19,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/436c04daa4ea443286b2f7b9b889341e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/436c04daa4ea443286b2f7b9b889341e 2024-11-19T16:15:19,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f6bdfd49042c4cc28094632eea440cd4 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/f6bdfd49042c4cc28094632eea440cd4 2024-11-19T16:15:19,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/af7584d3d9b94ceebfac478b8d819116, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/46551c4f955749249a791aca1792a12e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/e54e91fbbd2540d88e7b51c6e2b2a9e2, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/0b701aec8c7f4cb7a4e20908f872361f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a55d27423b0a43b98ad415c8a992ff57, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/52aea822467a447f9c5dd67b36ba6b08, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/86e11f7fa8054edab36e2727fe621a23, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/9a896c983d2e4f7a96e92c04f89757fe, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/f1c13857d8784cfaafa4d2fbe1af9d0f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a7318b053b98411d96e0e1729ea37663, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/aa76bd7eed5a41a0952887a0a3f90219, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/995cd29ae13f4887936aaab9eabede03, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/43b8c5b061e64bd0a22f391f158fffc1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/c10ec72ea0514b9f81c66ea5882f4c6d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/383e7f1417d44006ae433747fbcf8630, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/4c6c7cd8d25f45a39a8ac2d4e0e0a3cb, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/c8ac09e4ebec475d9e2131a99a45849c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/fabb99695cb041c0b24724800c634566, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/30f666be542f42e19e0fab4aaed211f3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/d88389efc0674dcf9f0e92b7c7238e46, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/ad754a22e00644918929670704aa4a11, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/99ab75ff5e9b4f78a6734924f2329501, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/7e7d784cf3b543fe9eb681569c9a2ad1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/ec1053c861944b238c4e2ef48527e1d9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/461b3c79e40b444f852b67ddf2d18a8c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a18e766f0f65400fb2d138f34d1fba6c] to archive 2024-11-19T16:15:19,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:15:19,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/af7584d3d9b94ceebfac478b8d819116 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/af7584d3d9b94ceebfac478b8d819116 2024-11-19T16:15:19,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/46551c4f955749249a791aca1792a12e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/46551c4f955749249a791aca1792a12e 2024-11-19T16:15:19,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/e54e91fbbd2540d88e7b51c6e2b2a9e2 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/e54e91fbbd2540d88e7b51c6e2b2a9e2 2024-11-19T16:15:19,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/0b701aec8c7f4cb7a4e20908f872361f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/0b701aec8c7f4cb7a4e20908f872361f 2024-11-19T16:15:19,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a55d27423b0a43b98ad415c8a992ff57 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a55d27423b0a43b98ad415c8a992ff57 2024-11-19T16:15:19,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/52aea822467a447f9c5dd67b36ba6b08 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/52aea822467a447f9c5dd67b36ba6b08 2024-11-19T16:15:19,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/86e11f7fa8054edab36e2727fe621a23 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/86e11f7fa8054edab36e2727fe621a23 2024-11-19T16:15:19,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/9a896c983d2e4f7a96e92c04f89757fe to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/9a896c983d2e4f7a96e92c04f89757fe 2024-11-19T16:15:19,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/f1c13857d8784cfaafa4d2fbe1af9d0f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/f1c13857d8784cfaafa4d2fbe1af9d0f 2024-11-19T16:15:19,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a7318b053b98411d96e0e1729ea37663 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a7318b053b98411d96e0e1729ea37663 2024-11-19T16:15:19,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/aa76bd7eed5a41a0952887a0a3f90219 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/aa76bd7eed5a41a0952887a0a3f90219 2024-11-19T16:15:19,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/995cd29ae13f4887936aaab9eabede03 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/995cd29ae13f4887936aaab9eabede03 2024-11-19T16:15:19,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/43b8c5b061e64bd0a22f391f158fffc1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/43b8c5b061e64bd0a22f391f158fffc1 2024-11-19T16:15:19,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/c10ec72ea0514b9f81c66ea5882f4c6d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/c10ec72ea0514b9f81c66ea5882f4c6d 2024-11-19T16:15:19,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/383e7f1417d44006ae433747fbcf8630 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/383e7f1417d44006ae433747fbcf8630 2024-11-19T16:15:19,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/4c6c7cd8d25f45a39a8ac2d4e0e0a3cb to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/4c6c7cd8d25f45a39a8ac2d4e0e0a3cb 2024-11-19T16:15:19,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/c8ac09e4ebec475d9e2131a99a45849c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/c8ac09e4ebec475d9e2131a99a45849c 2024-11-19T16:15:19,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/fabb99695cb041c0b24724800c634566 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/fabb99695cb041c0b24724800c634566 2024-11-19T16:15:19,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/30f666be542f42e19e0fab4aaed211f3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/30f666be542f42e19e0fab4aaed211f3 2024-11-19T16:15:19,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/d88389efc0674dcf9f0e92b7c7238e46 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/d88389efc0674dcf9f0e92b7c7238e46 2024-11-19T16:15:19,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/ad754a22e00644918929670704aa4a11 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/ad754a22e00644918929670704aa4a11 2024-11-19T16:15:19,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/99ab75ff5e9b4f78a6734924f2329501 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/99ab75ff5e9b4f78a6734924f2329501 2024-11-19T16:15:19,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/7e7d784cf3b543fe9eb681569c9a2ad1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/7e7d784cf3b543fe9eb681569c9a2ad1 2024-11-19T16:15:19,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/ec1053c861944b238c4e2ef48527e1d9 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/ec1053c861944b238c4e2ef48527e1d9 2024-11-19T16:15:19,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/461b3c79e40b444f852b67ddf2d18a8c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/461b3c79e40b444f852b67ddf2d18a8c 2024-11-19T16:15:19,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a18e766f0f65400fb2d138f34d1fba6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/a18e766f0f65400fb2d138f34d1fba6c 2024-11-19T16:15:19,167 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/recovered.edits/417.seqid, newMaxSeqId=417, maxSeqId=4 2024-11-19T16:15:19,168 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. 2024-11-19T16:15:19,168 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 828823ece6f79c9635b39bcf387a8d6c: Waiting for close lock at 1732032917376Running coprocessor pre-close hooks at 1732032917376Disabling compacts and flushes for region at 1732032917376Disabling writes for close at 1732032917377 (+1 ms)Obtaining lock to block concurrent updates at 1732032917377Preparing flush snapshotting stores in 828823ece6f79c9635b39bcf387a8d6c at 1732032917377Finished memstore snapshotting TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732032917378 (+1 ms)Flushing stores of TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c. at 1732032917378Flushing 828823ece6f79c9635b39bcf387a8d6c/A: creating writer at 1732032917379 (+1 ms)Flushing mob file 828823ece6f79c9635b39bcf387a8d6c/A: appending metadata at 1732032917387 (+8 ms)Flushing mob file 828823ece6f79c9635b39bcf387a8d6c/A: closing flushed file at 1732032917387Flushing 828823ece6f79c9635b39bcf387a8d6c/A: appending metadata at 1732032917808 (+421 ms)Flushing 828823ece6f79c9635b39bcf387a8d6c/A: closing flushed file at 1732032917808Flushing 828823ece6f79c9635b39bcf387a8d6c/B: creating writer at 1732032918224 (+416 ms)Flushing 828823ece6f79c9635b39bcf387a8d6c/B: appending metadata at 1732032918231 (+7 ms)Flushing 828823ece6f79c9635b39bcf387a8d6c/B: closing flushed file at 1732032918231Flushing 828823ece6f79c9635b39bcf387a8d6c/C: creating writer at 1732032918655 (+424 ms)Flushing 828823ece6f79c9635b39bcf387a8d6c/C: appending metadata at 1732032918658 (+3 ms)Flushing 828823ece6f79c9635b39bcf387a8d6c/C: closing flushed file at 1732032918658Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15713d6b: reopening flushed file at 1732032919071 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@381f0eb9: reopening flushed file at 1732032919079 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25c18ca4: reopening flushed file at 1732032919084 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 828823ece6f79c9635b39bcf387a8d6c in 1712ms, sequenceid=414, compaction requested=false at 1732032919089 (+5 ms)Writing region close event to WAL at 1732032919165 (+76 ms)Running coprocessor post-close hooks at 1732032919168 (+3 ms)Closed at 1732032919168 2024-11-19T16:15:19,170 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,170 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=828823ece6f79c9635b39bcf387a8d6c, regionState=CLOSED 2024-11-19T16:15:19,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:15:19,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-19T16:15:19,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 828823ece6f79c9635b39bcf387a8d6c, server=32a8860bb9d6,43095,1732032854050 in 1.9490 sec 2024-11-19T16:15:19,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-19T16:15:19,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=828823ece6f79c9635b39bcf387a8d6c, UNASSIGN in 1.9520 sec 2024-11-19T16:15:19,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-19T16:15:19,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9560 sec 2024-11-19T16:15:19,178 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032919178"}]},"ts":"1732032919178"} 2024-11-19T16:15:19,179 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T16:15:19,179 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T16:15:19,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9700 sec 2024-11-19T16:15:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T16:15:19,357 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:19,358 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-19T16:15:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:19,361 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T16:15:19,363 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:19,368 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,372 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/recovered.edits] 2024-11-19T16:15:19,376 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/585eb8cc81ed4e9f871cd3614db52869 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/585eb8cc81ed4e9f871cd3614db52869 2024-11-19T16:15:19,377 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/bba8680a33cf42ab9bc4a2c9eef64179 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/A/bba8680a33cf42ab9bc4a2c9eef64179 2024-11-19T16:15:19,379 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/92dfa32f0de745f183ab7e473d035e53 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/92dfa32f0de745f183ab7e473d035e53 2024-11-19T16:15:19,380 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/b90f3ad0ad6c4ac984cd1e74ca35e2e2 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/B/b90f3ad0ad6c4ac984cd1e74ca35e2e2 2024-11-19T16:15:19,382 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/2025b63a344345b58f2ab2cd60510c3c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/2025b63a344345b58f2ab2cd60510c3c 2024-11-19T16:15:19,383 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/7df4760cb8084f79b82fc786fd2aca99 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/C/7df4760cb8084f79b82fc786fd2aca99 2024-11-19T16:15:19,386 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/recovered.edits/417.seqid to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c/recovered.edits/417.seqid 2024-11-19T16:15:19,386 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,386 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T16:15:19,387 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T16:15:19,388 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T16:15:19,392 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190727b07384ef4c1ea2741347b964614c_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190727b07384ef4c1ea2741347b964614c_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,393 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111908079d55f5604a00b925d85bb34d389f_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111908079d55f5604a00b925d85bb34d389f_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,395 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119232d5b1b92684b90b61b9ae566c81e0c_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119232d5b1b92684b90b61b9ae566c81e0c_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,396 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193733b8c82fc9426bb86904d444026260_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193733b8c82fc9426bb86904d444026260_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,397 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119436413e77cc5400f8ef3275468fbedcf_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119436413e77cc5400f8ef3275468fbedcf_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,398 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194700052d96ba479d939654b847222be8_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194700052d96ba479d939654b847222be8_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,399 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119493f1bbcc3364d33b1f5525dbee316b9_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119493f1bbcc3364d33b1f5525dbee316b9_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,400 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194c834bf0d7194ebc8a447a0b50be9a36_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194c834bf0d7194ebc8a447a0b50be9a36_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,401 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111952e02d371035475bbf54235f2fbbbf09_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111952e02d371035475bbf54235f2fbbbf09_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,402 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111954ca1c18a4fd47eb93b86840847b258b_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111954ca1c18a4fd47eb93b86840847b258b_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,403 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195cc36e50ff6b4227ab757b6c6a9f8e03_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195cc36e50ff6b4227ab757b6c6a9f8e03_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,404 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119681543d23b2545fe8f158ad9ee56a211_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119681543d23b2545fe8f158ad9ee56a211_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,405 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111970551f9eed004330b72a77be7f8c7eb5_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111970551f9eed004330b72a77be7f8c7eb5_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,406 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119795ff68a42b34522ac22798e95b40a61_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119795ff68a42b34522ac22798e95b40a61_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,408 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198d2a4291ff6649a89a3d33cf9e7d390b_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198d2a4291ff6649a89a3d33cf9e7d390b_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,409 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c2baeba9f931423b854473c3f0ca4e1a_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c2baeba9f931423b854473c3f0ca4e1a_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,410 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e2d92001a5e2422a9804e7842616438a_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e2d92001a5e2422a9804e7842616438a_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,410 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119eae7c75f28604a198f475f386e47832d_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119eae7c75f28604a198f475f386e47832d_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,411 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ed71c66b2bfa47b09ad12b8bd68261ac_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ed71c66b2bfa47b09ad12b8bd68261ac_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,412 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f11d1f13ea174a3cacb39b2aecfbc975_828823ece6f79c9635b39bcf387a8d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f11d1f13ea174a3cacb39b2aecfbc975_828823ece6f79c9635b39bcf387a8d6c 2024-11-19T16:15:19,413 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T16:15:19,415 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:19,417 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T16:15:19,419 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T16:15:19,420 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:19,420 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T16:15:19,421 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732032919420"}]},"ts":"9223372036854775807"} 2024-11-19T16:15:19,423 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T16:15:19,423 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 828823ece6f79c9635b39bcf387a8d6c, NAME => 'TestAcidGuarantees,,1732032884977.828823ece6f79c9635b39bcf387a8d6c.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T16:15:19,423 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T16:15:19,423 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732032919423"}]},"ts":"9223372036854775807"} 2024-11-19T16:15:19,425 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T16:15:19,425 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:19,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-19T16:15:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T16:15:19,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T16:15:19,467 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:19,479 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=344 (was 377), ProcessCount=11 (was 11), AvailableMemoryMB=4172 (was 4383) 2024-11-19T16:15:19,489 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=344, ProcessCount=11, AvailableMemoryMB=4172 2024-11-19T16:15:19,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T16:15:19,491 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T16:15:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:19,493 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T16:15:19,493 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:19,493 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-19T16:15:19,494 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T16:15:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T16:15:19,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742066_1242 (size=1061) 2024-11-19T16:15:19,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T16:15:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T16:15:19,902 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70 2024-11-19T16:15:19,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742067_1243 (size=53) 2024-11-19T16:15:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T16:15:20,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:15:20,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1601148ee1f598abf2fc9cfff51f933e, disabling compactions & flushes 2024-11-19T16:15:20,310 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:20,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:20,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. after waiting 0 ms 2024-11-19T16:15:20,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:20,310 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:20,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1601148ee1f598abf2fc9cfff51f933e: Waiting for close lock at 1732032920310Disabling compacts and flushes for region at 1732032920310Disabling writes for close at 1732032920310Writing region close event to WAL at 1732032920310Closed at 1732032920310 2024-11-19T16:15:20,311 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T16:15:20,312 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732032920311"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732032920311"}]},"ts":"1732032920311"} 2024-11-19T16:15:20,313 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T16:15:20,314 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T16:15:20,315 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032920314"}]},"ts":"1732032920314"} 2024-11-19T16:15:20,316 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T16:15:20,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1601148ee1f598abf2fc9cfff51f933e, ASSIGN}] 2024-11-19T16:15:20,318 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1601148ee1f598abf2fc9cfff51f933e, ASSIGN 2024-11-19T16:15:20,319 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1601148ee1f598abf2fc9cfff51f933e, ASSIGN; state=OFFLINE, location=32a8860bb9d6,43095,1732032854050; forceNewPlan=false, retain=false 2024-11-19T16:15:20,470 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1601148ee1f598abf2fc9cfff51f933e, regionState=OPENING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:20,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1601148ee1f598abf2fc9cfff51f933e, ASSIGN because future has completed 2024-11-19T16:15:20,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:15:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T16:15:20,627 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:20,627 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} 2024-11-19T16:15:20,627 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:20,628 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:15:20,628 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:20,628 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:20,629 INFO [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:20,631 INFO [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:15:20,631 INFO [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1601148ee1f598abf2fc9cfff51f933e columnFamilyName A 2024-11-19T16:15:20,631 DEBUG [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:20,632 INFO [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] regionserver.HStore(327): Store=1601148ee1f598abf2fc9cfff51f933e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:15:20,632 INFO [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:20,633 INFO [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:15:20,634 INFO [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1601148ee1f598abf2fc9cfff51f933e columnFamilyName B 2024-11-19T16:15:20,634 DEBUG [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:20,634 INFO [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] regionserver.HStore(327): Store=1601148ee1f598abf2fc9cfff51f933e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:15:20,634 INFO [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:20,636 INFO [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:15:20,636 INFO [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1601148ee1f598abf2fc9cfff51f933e columnFamilyName C 2024-11-19T16:15:20,636 DEBUG [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:20,636 INFO [StoreOpener-1601148ee1f598abf2fc9cfff51f933e-1 {}] regionserver.HStore(327): Store=1601148ee1f598abf2fc9cfff51f933e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:15:20,637 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:20,637 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:20,638 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:20,638 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:20,639 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:20,639 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:20,640 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T16:15:20,641 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:20,644 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T16:15:20,645 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 1601148ee1f598abf2fc9cfff51f933e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64657748, jitterRate=-0.03652447462081909}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T16:15:20,645 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:20,646 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 1601148ee1f598abf2fc9cfff51f933e: Running coprocessor pre-open hook at 1732032920628Writing region info on filesystem at 1732032920628Initializing all the Stores at 1732032920629 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032920629Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032920629Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032920629Cleaning up temporary data from old regions at 1732032920639 (+10 ms)Running coprocessor post-open hooks at 1732032920645 (+6 ms)Region opened successfully at 1732032920646 (+1 ms) 2024-11-19T16:15:20,647 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., pid=61, masterSystemTime=1732032920623 2024-11-19T16:15:20,649 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:20,649 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:20,650 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1601148ee1f598abf2fc9cfff51f933e, regionState=OPEN, openSeqNum=2, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:20,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:15:20,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-19T16:15:20,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 in 180 msec 2024-11-19T16:15:20,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-19T16:15:20,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1601148ee1f598abf2fc9cfff51f933e, ASSIGN in 337 msec 2024-11-19T16:15:20,662 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T16:15:20,662 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032920662"}]},"ts":"1732032920662"} 2024-11-19T16:15:20,664 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T16:15:20,665 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T16:15:20,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-19T16:15:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T16:15:21,637 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:21,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,638 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:21,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:21,640 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41af849a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:21,642 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@867404c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,643 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:21,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:21,646 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T16:15:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 32a8860bb9d6,37355,1732032853237 2024-11-19T16:15:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@69bc622c 2024-11-19T16:15:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T16:15:21,648 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T16:15:21,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:21,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:21,650 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:21,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:21,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb7a00d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:21,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:21,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:21,652 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3ebda6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:21,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:21,654 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b009d75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:21,655 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25a85f5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:21,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:21,657 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b9bf543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:21,658 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:21,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af87c71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:21,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:21,660 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a05258, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:21,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:21,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:21,661 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@796d1463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:21,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:21,663 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ae1e07a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:21,665 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11bcdaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:21,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:21,667 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@286b99d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:21,669 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148b2368, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:21,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:21,671 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2251037e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:21,673 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8f0e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:21,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:21,678 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@91f5ec7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:21,680 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51efc443, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:21,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:21,683 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ade7820, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:21,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d2c5832, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:21,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:21,688 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@431c0b45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:21,690 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:21,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f9f51c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:21,694 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:21,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:15:21,703 DEBUG [Thread-1133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:21,704 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:21,704 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:21,704 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:21,704 DEBUG [Thread-1140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:21,704 DEBUG [Thread-1142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-19T16:15:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:21,705 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:21,705 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:21,707 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T16:15:21,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:15:21,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:21,708 DEBUG [Thread-1129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:21,708 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:21,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:15:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:15:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:15:21,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:15:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:21,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:21,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:15:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:21,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:15:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:21,721 DEBUG [Thread-1144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:21,722 DEBUG [Thread-1131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:21,724 DEBUG [Thread-1135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:21,724 DEBUG [Thread-1135 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:21,725 DEBUG [Thread-1135 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:21,725 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:21,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:21,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:15:21,729 DEBUG [Thread-1135 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:15:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:15:21,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/2885864f6fd14d26bbbb43b3754aee52 is 50, key is test_row_0/A:col10/1732032921719/Put/seqid=0 2024-11-19T16:15:21,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732032921738, max timestamp=1732032921753Num uniques 859; 2024-11-19T16:15:21,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T16:15:21,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032921749, max timestamp=1732032921755Num uniques 859; 2024-11-19T16:15:21,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032921749, max timestamp=1732032921755Num uniques 859; 2024-11-19T16:15:21,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:15:21,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:15:21,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742068_1244 (size=9657) 2024-11-19T16:15:21,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/2885864f6fd14d26bbbb43b3754aee52 2024-11-19T16:15:21,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032981790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:21,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032981799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:21,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032981805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:21,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032981805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:21,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032981799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T16:15:21,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/b8bf7fb557f745538865bc94ee06ada7 is 50, key is test_row_0/B:col10/1732032921719/Put/seqid=0 2024-11-19T16:15:21,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T16:15:21,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:21,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:21,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:21,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:21,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:21,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742069_1245 (size=9657) 2024-11-19T16:15:21,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032981907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:21,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032981917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:21,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:21,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032981917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032981917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:21,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032981917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T16:15:22,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:22,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T16:15:22,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032982117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032982127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032982127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032982127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032982127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T16:15:22,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:22,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/b8bf7fb557f745538865bc94ee06ada7 2024-11-19T16:15:22,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/72204856534d4f059fbc8753db93b207 is 50, key is test_row_0/C:col10/1732032921719/Put/seqid=0 2024-11-19T16:15:22,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742070_1246 (size=9657) 2024-11-19T16:15:22,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/72204856534d4f059fbc8753db93b207 2024-11-19T16:15:22,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T16:15:22,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:22,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/2885864f6fd14d26bbbb43b3754aee52 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/2885864f6fd14d26bbbb43b3754aee52 2024-11-19T16:15:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T16:15:22,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/2885864f6fd14d26bbbb43b3754aee52, entries=100, sequenceid=11, filesize=9.4 K 2024-11-19T16:15:22,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/b8bf7fb557f745538865bc94ee06ada7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/b8bf7fb557f745538865bc94ee06ada7 2024-11-19T16:15:22,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/b8bf7fb557f745538865bc94ee06ada7, entries=100, sequenceid=11, filesize=9.4 K 2024-11-19T16:15:22,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/72204856534d4f059fbc8753db93b207 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/72204856534d4f059fbc8753db93b207 2024-11-19T16:15:22,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/72204856534d4f059fbc8753db93b207, entries=100, sequenceid=11, filesize=9.4 K 2024-11-19T16:15:22,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1601148ee1f598abf2fc9cfff51f933e in 640ms, sequenceid=11, compaction requested=false 2024-11-19T16:15:22,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:22,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-19T16:15:22,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:22,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/24d991694b03481eb8a117a905eb5b9f is 50, key is test_row_0/A:col10/1732032921799/Put/seqid=0 2024-11-19T16:15:22,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032982440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032982442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032982442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032982443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032982443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742071_1247 (size=12001) 2024-11-19T16:15:22,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T16:15:22,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:22,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032982546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032982547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032982547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032982557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032982557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T16:15:22,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:22,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032982757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032982757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032982757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032982767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032982767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:22,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T16:15:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T16:15:22,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/24d991694b03481eb8a117a905eb5b9f 2024-11-19T16:15:22,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/86fe69b828524701a2452b2e79ed0e3c is 50, key is test_row_0/B:col10/1732032921799/Put/seqid=0 2024-11-19T16:15:22,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742072_1248 (size=12001) 2024-11-19T16:15:22,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/86fe69b828524701a2452b2e79ed0e3c 2024-11-19T16:15:22,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/2de55cd250ce461c9887122349326132 is 50, key is test_row_0/C:col10/1732032921799/Put/seqid=0 2024-11-19T16:15:22,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T16:15:22,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:22,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:22,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:22,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742073_1249 (size=12001) 2024-11-19T16:15:23,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032983067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032983067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032983067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032983076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032983076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T16:15:23,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:23,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:23,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:23,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:23,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:23,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T16:15:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:23,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:23,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:23,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/2de55cd250ce461c9887122349326132 2024-11-19T16:15:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/24d991694b03481eb8a117a905eb5b9f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/24d991694b03481eb8a117a905eb5b9f 2024-11-19T16:15:23,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/24d991694b03481eb8a117a905eb5b9f, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T16:15:23,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/86fe69b828524701a2452b2e79ed0e3c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/86fe69b828524701a2452b2e79ed0e3c 2024-11-19T16:15:23,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/86fe69b828524701a2452b2e79ed0e3c, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T16:15:23,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/2de55cd250ce461c9887122349326132 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2de55cd250ce461c9887122349326132 2024-11-19T16:15:23,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2de55cd250ce461c9887122349326132, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T16:15:23,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 1601148ee1f598abf2fc9cfff51f933e in 955ms, sequenceid=48, compaction requested=false 2024-11-19T16:15:23,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:23,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T16:15:23,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:23,413 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:15:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:23,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/870a35566da047d29921cf9681d3dafe is 50, key is test_row_0/A:col10/1732032922441/Put/seqid=0 2024-11-19T16:15:23,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742074_1250 (size=12001) 2024-11-19T16:15:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:23,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:23,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032923598, max timestamp=1732032923605Num uniques 859; 2024-11-19T16:15:23,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:15:23,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732032923600, max timestamp=1732032923605Num uniques 859; 2024-11-19T16:15:23,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T16:15:23,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732032923600, max timestamp=1732032923605Num uniques 859; 2024-11-19T16:15:23,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T16:15:23,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032983629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032983630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032983631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032983632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032983633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032983747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032983747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032983747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032983747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032983747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,835 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/870a35566da047d29921cf9681d3dafe 2024-11-19T16:15:23,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/29dde8f15794404398996ffe9a9ce764 is 50, key is test_row_0/B:col10/1732032922441/Put/seqid=0 2024-11-19T16:15:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T16:15:23,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742075_1251 (size=12001) 2024-11-19T16:15:23,875 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/29dde8f15794404398996ffe9a9ce764 2024-11-19T16:15:23,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/21fd218e3e0f4be89cef6bba4bbed1b7 is 50, key is test_row_0/C:col10/1732032922441/Put/seqid=0 2024-11-19T16:15:23,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742076_1252 (size=12001) 2024-11-19T16:15:23,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032983956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032983957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032983957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032983957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032983957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032984266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032984267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032984267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032984267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032984267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,316 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/21fd218e3e0f4be89cef6bba4bbed1b7 2024-11-19T16:15:24,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/870a35566da047d29921cf9681d3dafe as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/870a35566da047d29921cf9681d3dafe 2024-11-19T16:15:24,342 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/870a35566da047d29921cf9681d3dafe, entries=150, sequenceid=58, filesize=11.7 K 2024-11-19T16:15:24,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/29dde8f15794404398996ffe9a9ce764 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/29dde8f15794404398996ffe9a9ce764 2024-11-19T16:15:24,350 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/29dde8f15794404398996ffe9a9ce764, entries=150, sequenceid=58, filesize=11.7 K 2024-11-19T16:15:24,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/21fd218e3e0f4be89cef6bba4bbed1b7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/21fd218e3e0f4be89cef6bba4bbed1b7 2024-11-19T16:15:24,368 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/21fd218e3e0f4be89cef6bba4bbed1b7, entries=150, sequenceid=58, filesize=11.7 K 2024-11-19T16:15:24,370 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=137.86 KB/141170 for 1601148ee1f598abf2fc9cfff51f933e in 957ms, sequenceid=58, compaction requested=true 2024-11-19T16:15:24,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:24,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:24,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-19T16:15:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-19T16:15:24,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-19T16:15:24,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6630 sec 2024-11-19T16:15:24,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.6730 sec 2024-11-19T16:15:24,706 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T16:15:24,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=144.57 KB heapSize=413.94 KB 2024-11-19T16:15:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T16:15:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T16:15:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T16:15:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:24,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/822c166c429f44f1b23404f0351de81a is 50, key is test_row_0/A:col10/1732032924777/Put/seqid=0 2024-11-19T16:15:24,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032984782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032984782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032984786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032984788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032984789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742077_1253 (size=12001) 2024-11-19T16:15:24,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032984897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032984897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032984897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032984897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:24,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032984906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:25,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032985107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032985107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032985107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032985107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:25,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032985116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:25,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/822c166c429f44f1b23404f0351de81a 2024-11-19T16:15:25,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/c75c1d4bfab74dc1a747f4f3c6c514e2 is 50, key is test_row_0/B:col10/1732032924777/Put/seqid=0 2024-11-19T16:15:25,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742078_1254 (size=12001) 2024-11-19T16:15:25,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032985417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032985417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032985417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:25,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032985417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:25,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032985426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:25,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/c75c1d4bfab74dc1a747f4f3c6c514e2 2024-11-19T16:15:25,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/8436541e846a4eb8820da52ca630112a is 50, key is test_row_0/C:col10/1732032924777/Put/seqid=0 2024-11-19T16:15:25,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742079_1255 (size=12001) 2024-11-19T16:15:25,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/8436541e846a4eb8820da52ca630112a 2024-11-19T16:15:25,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/822c166c429f44f1b23404f0351de81a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/822c166c429f44f1b23404f0351de81a 2024-11-19T16:15:25,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/822c166c429f44f1b23404f0351de81a, entries=150, sequenceid=96, filesize=11.7 K 2024-11-19T16:15:25,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/c75c1d4bfab74dc1a747f4f3c6c514e2 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/c75c1d4bfab74dc1a747f4f3c6c514e2 2024-11-19T16:15:25,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/c75c1d4bfab74dc1a747f4f3c6c514e2, entries=150, sequenceid=96, filesize=11.7 K 2024-11-19T16:15:25,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/8436541e846a4eb8820da52ca630112a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/8436541e846a4eb8820da52ca630112a 2024-11-19T16:15:25,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/8436541e846a4eb8820da52ca630112a, entries=150, sequenceid=96, filesize=11.7 K 2024-11-19T16:15:25,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~144.57 KB/148040, heapSize ~413.89 KB/423824, currentSize=33.54 KB/34350 for 1601148ee1f598abf2fc9cfff51f933e in 976ms, sequenceid=96, compaction requested=true 2024-11-19T16:15:25,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:25,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:25,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:25,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:25,755 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:25,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:25,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:25,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T16:15:25,756 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:25,757 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:25,757 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/B is initiating minor compaction (all files) 2024-11-19T16:15:25,757 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/B in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:25,757 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/b8bf7fb557f745538865bc94ee06ada7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/86fe69b828524701a2452b2e79ed0e3c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/29dde8f15794404398996ffe9a9ce764, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/c75c1d4bfab74dc1a747f4f3c6c514e2] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=44.6 K 2024-11-19T16:15:25,758 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8bf7fb557f745538865bc94ee06ada7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732032921716 2024-11-19T16:15:25,758 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:25,758 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/A is initiating minor compaction (all files) 2024-11-19T16:15:25,758 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/A in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:25,758 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/2885864f6fd14d26bbbb43b3754aee52, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/24d991694b03481eb8a117a905eb5b9f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/870a35566da047d29921cf9681d3dafe, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/822c166c429f44f1b23404f0351de81a] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=44.6 K 2024-11-19T16:15:25,759 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86fe69b828524701a2452b2e79ed0e3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732032921785 2024-11-19T16:15:25,759 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 2885864f6fd14d26bbbb43b3754aee52, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732032921716 2024-11-19T16:15:25,759 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29dde8f15794404398996ffe9a9ce764, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032922440 2024-11-19T16:15:25,759 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 24d991694b03481eb8a117a905eb5b9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732032921785 2024-11-19T16:15:25,760 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting c75c1d4bfab74dc1a747f4f3c6c514e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732032923631 2024-11-19T16:15:25,760 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 870a35566da047d29921cf9681d3dafe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032922440 2024-11-19T16:15:25,760 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 822c166c429f44f1b23404f0351de81a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732032923631 2024-11-19T16:15:25,783 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#B#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:25,784 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/531b04dcfaa5407ba20612277f134a89 is 50, key is test_row_0/B:col10/1732032924777/Put/seqid=0 2024-11-19T16:15:25,789 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#A#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:25,789 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/509e5cac03434c95bd3bab39849ea1e1 is 50, key is test_row_0/A:col10/1732032924777/Put/seqid=0 2024-11-19T16:15:25,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742081_1257 (size=12139) 2024-11-19T16:15:25,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742080_1256 (size=12139) 2024-11-19T16:15:25,857 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/531b04dcfaa5407ba20612277f134a89 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/531b04dcfaa5407ba20612277f134a89 2024-11-19T16:15:25,870 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/509e5cac03434c95bd3bab39849ea1e1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/509e5cac03434c95bd3bab39849ea1e1 2024-11-19T16:15:25,871 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/B of 1601148ee1f598abf2fc9cfff51f933e into 531b04dcfaa5407ba20612277f134a89(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:25,871 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:25,871 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/B, priority=12, startTime=1732032925755; duration=0sec 2024-11-19T16:15:25,871 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:25,871 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:B 2024-11-19T16:15:25,871 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:25,874 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:25,874 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/C is initiating minor compaction (all files) 2024-11-19T16:15:25,874 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/C in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:25,875 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/72204856534d4f059fbc8753db93b207, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2de55cd250ce461c9887122349326132, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/21fd218e3e0f4be89cef6bba4bbed1b7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/8436541e846a4eb8820da52ca630112a] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=44.6 K 2024-11-19T16:15:25,875 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72204856534d4f059fbc8753db93b207, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732032921716 2024-11-19T16:15:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T16:15:25,878 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:25,878 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:25,878 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/A of 1601148ee1f598abf2fc9cfff51f933e into 509e5cac03434c95bd3bab39849ea1e1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:25,879 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:25,879 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/A, priority=12, startTime=1732032925755; duration=0sec 2024-11-19T16:15:25,879 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:25,879 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:A 2024-11-19T16:15:25,879 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2de55cd250ce461c9887122349326132, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732032921785 2024-11-19T16:15:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-19T16:15:25,880 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21fd218e3e0f4be89cef6bba4bbed1b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032922440 2024-11-19T16:15:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T16:15:25,880 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:25,881 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8436541e846a4eb8820da52ca630112a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732032923631 2024-11-19T16:15:25,881 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:25,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:25,911 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:25,911 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/911f44fedc7745d8a806850ece8500d8 is 50, key is test_row_0/C:col10/1732032924777/Put/seqid=0 2024-11-19T16:15:25,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:15:25,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:25,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:25,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:25,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:25,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:25,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:25,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:25,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742082_1258 (size=12139) 2024-11-19T16:15:25,963 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/911f44fedc7745d8a806850ece8500d8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/911f44fedc7745d8a806850ece8500d8 2024-11-19T16:15:25,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032925948, max timestamp=1732032925963Num uniques 859; 2024-11-19T16:15:25,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:15:25,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032925948, max timestamp=1732032925963Num uniques 859; 2024-11-19T16:15:25,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:15:25,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032925948, max timestamp=1732032925963Num uniques 859; 2024-11-19T16:15:25,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:15:25,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/381324ed460a4a818fc5122f99fde1bf is 50, key is test_row_0/A:col10/1732032924786/Put/seqid=0 2024-11-19T16:15:25,978 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/C of 1601148ee1f598abf2fc9cfff51f933e into 911f44fedc7745d8a806850ece8500d8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:25,978 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:25,978 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/C, priority=12, startTime=1732032925755; duration=0sec 2024-11-19T16:15:25,979 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:25,979 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:C 2024-11-19T16:15:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T16:15:26,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032985997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032985999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742083_1259 (size=16681) 2024-11-19T16:15:26,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/381324ed460a4a818fc5122f99fde1bf 2024-11-19T16:15:26,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032986004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032986007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032986002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/0d73916bb0b84089b333344aa0b3862e is 50, key is test_row_0/B:col10/1732032924786/Put/seqid=0 2024-11-19T16:15:26,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T16:15:26,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:26,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:26,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:26,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:26,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:26,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742084_1260 (size=12001) 2024-11-19T16:15:26,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/0d73916bb0b84089b333344aa0b3862e 2024-11-19T16:15:26,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/20b45d8e5c4c49a891a338e2300a244f is 50, key is test_row_0/C:col10/1732032924786/Put/seqid=0 2024-11-19T16:15:26,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032986117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032986117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032986117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742085_1261 (size=12001) 2024-11-19T16:15:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/20b45d8e5c4c49a891a338e2300a244f 2024-11-19T16:15:26,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/381324ed460a4a818fc5122f99fde1bf as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/381324ed460a4a818fc5122f99fde1bf 2024-11-19T16:15:26,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032986127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032986127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/381324ed460a4a818fc5122f99fde1bf, entries=250, sequenceid=109, filesize=16.3 K 2024-11-19T16:15:26,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/0d73916bb0b84089b333344aa0b3862e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0d73916bb0b84089b333344aa0b3862e 2024-11-19T16:15:26,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0d73916bb0b84089b333344aa0b3862e, entries=150, sequenceid=109, filesize=11.7 K 2024-11-19T16:15:26,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/20b45d8e5c4c49a891a338e2300a244f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/20b45d8e5c4c49a891a338e2300a244f 2024-11-19T16:15:26,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/20b45d8e5c4c49a891a338e2300a244f, entries=150, sequenceid=109, filesize=11.7 K 2024-11-19T16:15:26,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 1601148ee1f598abf2fc9cfff51f933e in 227ms, sequenceid=109, compaction requested=false 2024-11-19T16:15:26,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:26,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T16:15:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:26,189 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-11-19T16:15:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T16:15:26,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/9f3789adff6f4b28a09a3d59da2c85a6 is 50, key is test_row_0/A:col10/1732032926002/Put/seqid=0 2024-11-19T16:15:26,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742086_1262 (size=12151) 2024-11-19T16:15:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:26,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:26,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032986338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032986340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032986344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032986346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032986347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032986456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032986457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032986457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032986457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032986457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T16:15:26,621 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/9f3789adff6f4b28a09a3d59da2c85a6 2024-11-19T16:15:26,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/177265ec7fb642f79bd8662505bd65d7 is 50, key is test_row_0/B:col10/1732032926002/Put/seqid=0 2024-11-19T16:15:26,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742087_1263 (size=12151) 2024-11-19T16:15:26,639 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/177265ec7fb642f79bd8662505bd65d7 2024-11-19T16:15:26,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/89451ae47ea74bc8b5a8cefd59c63644 is 50, key is test_row_0/C:col10/1732032926002/Put/seqid=0 2024-11-19T16:15:26,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032986667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032986667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032986667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032986667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032986667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:26,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742088_1264 (size=12151) 2024-11-19T16:15:26,674 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/89451ae47ea74bc8b5a8cefd59c63644 2024-11-19T16:15:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/9f3789adff6f4b28a09a3d59da2c85a6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9f3789adff6f4b28a09a3d59da2c85a6 2024-11-19T16:15:26,688 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9f3789adff6f4b28a09a3d59da2c85a6, entries=150, sequenceid=145, filesize=11.9 K 2024-11-19T16:15:26,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/177265ec7fb642f79bd8662505bd65d7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/177265ec7fb642f79bd8662505bd65d7 2024-11-19T16:15:26,697 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/177265ec7fb642f79bd8662505bd65d7, entries=150, sequenceid=145, filesize=11.9 K 2024-11-19T16:15:26,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/89451ae47ea74bc8b5a8cefd59c63644 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/89451ae47ea74bc8b5a8cefd59c63644 2024-11-19T16:15:26,704 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/89451ae47ea74bc8b5a8cefd59c63644, entries=150, sequenceid=145, filesize=11.9 K 2024-11-19T16:15:26,706 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~365.41 KB/374176, currentSize=60.38 KB/61830 for 1601148ee1f598abf2fc9cfff51f933e in 517ms, sequenceid=145, compaction requested=true 2024-11-19T16:15:26,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:26,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:26,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-19T16:15:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-19T16:15:26,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-19T16:15:26,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 827 msec 2024-11-19T16:15:26,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 833 msec 2024-11-19T16:15:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:26,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T16:15:26,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:26,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:26,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:26,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:26,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:26,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:26,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/09f25293ba824742b39a466494f1aa7f is 50, key is test_row_0/A:col10/1732032926980/Put/seqid=0 2024-11-19T16:15:27,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032986997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032986997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032986998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032987000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032987002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T16:15:27,017 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:27,017 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:27,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742089_1265 (size=9757) 2024-11-19T16:15:27,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/09f25293ba824742b39a466494f1aa7f 2024-11-19T16:15:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-19T16:15:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T16:15:27,021 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:27,024 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:27,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:27,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/f81364e113744b45af0fa11f806333b6 is 50, key is test_row_0/B:col10/1732032926980/Put/seqid=0 2024-11-19T16:15:27,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742090_1266 (size=9757) 2024-11-19T16:15:27,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/f81364e113744b45af0fa11f806333b6 2024-11-19T16:15:27,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/e6086d9b81264e689bfc758a16930c5d is 50, key is test_row_0/C:col10/1732032926980/Put/seqid=0 2024-11-19T16:15:27,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742091_1267 (size=9757) 2024-11-19T16:15:27,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/e6086d9b81264e689bfc758a16930c5d 2024-11-19T16:15:27,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/09f25293ba824742b39a466494f1aa7f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/09f25293ba824742b39a466494f1aa7f 2024-11-19T16:15:27,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/09f25293ba824742b39a466494f1aa7f, entries=100, sequenceid=161, filesize=9.5 K 2024-11-19T16:15:27,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/f81364e113744b45af0fa11f806333b6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/f81364e113744b45af0fa11f806333b6 2024-11-19T16:15:27,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/f81364e113744b45af0fa11f806333b6, entries=100, sequenceid=161, filesize=9.5 K 2024-11-19T16:15:27,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/e6086d9b81264e689bfc758a16930c5d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/e6086d9b81264e689bfc758a16930c5d 2024-11-19T16:15:27,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032987107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032987107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032987107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032987107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/e6086d9b81264e689bfc758a16930c5d, entries=100, sequenceid=161, filesize=9.5 K 2024-11-19T16:15:27,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1601148ee1f598abf2fc9cfff51f933e in 131ms, sequenceid=161, compaction requested=true 2024-11-19T16:15:27,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:27,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:27,113 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:27,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:27,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:27,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:27,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:27,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:27,113 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:27,114 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:27,114 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/B is initiating minor compaction (all files) 2024-11-19T16:15:27,114 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/B in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:27,115 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/531b04dcfaa5407ba20612277f134a89, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0d73916bb0b84089b333344aa0b3862e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/177265ec7fb642f79bd8662505bd65d7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/f81364e113744b45af0fa11f806333b6] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=45.0 K 2024-11-19T16:15:27,115 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:27,115 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/A is initiating minor compaction (all files) 2024-11-19T16:15:27,115 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/A in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:27,115 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/509e5cac03434c95bd3bab39849ea1e1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/381324ed460a4a818fc5122f99fde1bf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9f3789adff6f4b28a09a3d59da2c85a6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/09f25293ba824742b39a466494f1aa7f] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=49.5 K 2024-11-19T16:15:27,115 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 531b04dcfaa5407ba20612277f134a89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732032923631 2024-11-19T16:15:27,116 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 509e5cac03434c95bd3bab39849ea1e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732032923631 2024-11-19T16:15:27,116 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d73916bb0b84089b333344aa0b3862e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732032924786 2024-11-19T16:15:27,116 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 381324ed460a4a818fc5122f99fde1bf, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732032924782 2024-11-19T16:15:27,116 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 177265ec7fb642f79bd8662505bd65d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732032925981 2024-11-19T16:15:27,117 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f3789adff6f4b28a09a3d59da2c85a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732032925981 2024-11-19T16:15:27,117 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting f81364e113744b45af0fa11f806333b6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732032926980 2024-11-19T16:15:27,117 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09f25293ba824742b39a466494f1aa7f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732032926980 2024-11-19T16:15:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:27,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T16:15:27,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:27,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:27,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:27,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:27,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:27,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T16:15:27,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/30cfe8187f094756bb00c2f806eb343b is 50, key is test_row_0/A:col10/1732032927118/Put/seqid=0 2024-11-19T16:15:27,139 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#B#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:27,139 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#A#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:27,139 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/69821af53a534e16b12e38c72dffceca is 50, key is test_row_0/B:col10/1732032926980/Put/seqid=0 2024-11-19T16:15:27,139 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/a99b48c8a8354efb806b9bd93f37c6b9 is 50, key is test_row_0/A:col10/1732032926980/Put/seqid=0 2024-11-19T16:15:27,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742092_1268 (size=14541) 2024-11-19T16:15:27,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/30cfe8187f094756bb00c2f806eb343b 2024-11-19T16:15:27,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/5187643330584e9d969035483ce3c3b3 is 50, key is test_row_0/B:col10/1732032927118/Put/seqid=0 2024-11-19T16:15:27,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742093_1269 (size=12425) 2024-11-19T16:15:27,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T16:15:27,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:27,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:27,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:27,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:27,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:27,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742094_1270 (size=12425) 2024-11-19T16:15:27,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742095_1271 (size=12151) 2024-11-19T16:15:27,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032987200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032987307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032987317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032987317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032987317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032987317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T16:15:27,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:27,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:27,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:27,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:27,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T16:15:27,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T16:15:27,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:27,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:27,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032987517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,572 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/a99b48c8a8354efb806b9bd93f37c6b9 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/a99b48c8a8354efb806b9bd93f37c6b9 2024-11-19T16:15:27,579 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/A of 1601148ee1f598abf2fc9cfff51f933e into a99b48c8a8354efb806b9bd93f37c6b9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:27,579 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:27,579 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/A, priority=12, startTime=1732032927113; duration=0sec 2024-11-19T16:15:27,579 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:27,579 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:A 2024-11-19T16:15:27,579 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:27,580 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:27,580 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/C is initiating minor compaction (all files) 2024-11-19T16:15:27,580 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/C in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:27,581 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/911f44fedc7745d8a806850ece8500d8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/20b45d8e5c4c49a891a338e2300a244f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/89451ae47ea74bc8b5a8cefd59c63644, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/e6086d9b81264e689bfc758a16930c5d] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=45.0 K 2024-11-19T16:15:27,581 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 911f44fedc7745d8a806850ece8500d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732032923631 2024-11-19T16:15:27,581 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20b45d8e5c4c49a891a338e2300a244f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732032924786 2024-11-19T16:15:27,581 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89451ae47ea74bc8b5a8cefd59c63644, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732032925981 2024-11-19T16:15:27,582 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6086d9b81264e689bfc758a16930c5d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732032926980 2024-11-19T16:15:27,589 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#C#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:27,590 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/3c21b8fd8cc64d93847a1be8debb5f2e is 50, key is test_row_0/C:col10/1732032926980/Put/seqid=0 2024-11-19T16:15:27,591 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/69821af53a534e16b12e38c72dffceca as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/69821af53a534e16b12e38c72dffceca 2024-11-19T16:15:27,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/5187643330584e9d969035483ce3c3b3 2024-11-19T16:15:27,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742096_1272 (size=12425) 2024-11-19T16:15:27,597 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/B of 1601148ee1f598abf2fc9cfff51f933e into 69821af53a534e16b12e38c72dffceca(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:27,597 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:27,597 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/B, priority=12, startTime=1732032927113; duration=0sec 2024-11-19T16:15:27,597 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:27,597 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:B 2024-11-19T16:15:27,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/7e636a9cb5d9415ba5b0c2c136a2ac29 is 50, key is test_row_0/C:col10/1732032927118/Put/seqid=0 2024-11-19T16:15:27,602 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/3c21b8fd8cc64d93847a1be8debb5f2e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/3c21b8fd8cc64d93847a1be8debb5f2e 2024-11-19T16:15:27,607 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/C of 1601148ee1f598abf2fc9cfff51f933e into 3c21b8fd8cc64d93847a1be8debb5f2e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:27,607 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:27,607 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/C, priority=12, startTime=1732032927113; duration=0sec 2024-11-19T16:15:27,607 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:27,607 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:C 2024-11-19T16:15:27,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742097_1273 (size=12151) 2024-11-19T16:15:27,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/7e636a9cb5d9415ba5b0c2c136a2ac29 2024-11-19T16:15:27,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/30cfe8187f094756bb00c2f806eb343b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/30cfe8187f094756bb00c2f806eb343b 2024-11-19T16:15:27,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/30cfe8187f094756bb00c2f806eb343b, entries=200, sequenceid=178, filesize=14.2 K 2024-11-19T16:15:27,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/5187643330584e9d969035483ce3c3b3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/5187643330584e9d969035483ce3c3b3 2024-11-19T16:15:27,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/5187643330584e9d969035483ce3c3b3, entries=150, sequenceid=178, filesize=11.9 K 2024-11-19T16:15:27,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/7e636a9cb5d9415ba5b0c2c136a2ac29 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/7e636a9cb5d9415ba5b0c2c136a2ac29 2024-11-19T16:15:27,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032987627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032987627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032987627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032987627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:27,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/7e636a9cb5d9415ba5b0c2c136a2ac29, entries=150, sequenceid=178, filesize=11.9 K 2024-11-19T16:15:27,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1601148ee1f598abf2fc9cfff51f933e in 510ms, sequenceid=178, compaction requested=false 2024-11-19T16:15:27,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:27,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T16:15:27,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:27,637 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T16:15:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:27,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/117a37d48bf44e9289ce9254929dd797 is 50, key is test_row_0/A:col10/1732032927193/Put/seqid=0 2024-11-19T16:15:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T16:15:27,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742098_1274 (size=12151) 2024-11-19T16:15:27,664 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/117a37d48bf44e9289ce9254929dd797 2024-11-19T16:15:27,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/0764508a07864a5a8f5a21714a49bfe0 is 50, key is test_row_0/B:col10/1732032927193/Put/seqid=0 2024-11-19T16:15:27,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742099_1275 (size=12151) 2024-11-19T16:15:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:27,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:27,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032987906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:28,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032988016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:28,114 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/0764508a07864a5a8f5a21714a49bfe0 2024-11-19T16:15:28,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/acbf4b171c5f47048a347830c8530f8c is 50, key is test_row_0/C:col10/1732032927193/Put/seqid=0 2024-11-19T16:15:28,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742100_1276 (size=12151) 2024-11-19T16:15:28,134 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/acbf4b171c5f47048a347830c8530f8c 2024-11-19T16:15:28,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032988137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032988137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:28,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032988137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:28,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032988137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:28,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/117a37d48bf44e9289ce9254929dd797 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/117a37d48bf44e9289ce9254929dd797 2024-11-19T16:15:28,147 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/117a37d48bf44e9289ce9254929dd797, entries=150, sequenceid=196, filesize=11.9 K 2024-11-19T16:15:28,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/0764508a07864a5a8f5a21714a49bfe0 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0764508a07864a5a8f5a21714a49bfe0 2024-11-19T16:15:28,153 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0764508a07864a5a8f5a21714a49bfe0, entries=150, sequenceid=196, filesize=11.9 K 2024-11-19T16:15:28,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/acbf4b171c5f47048a347830c8530f8c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/acbf4b171c5f47048a347830c8530f8c 2024-11-19T16:15:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T16:15:28,157 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/acbf4b171c5f47048a347830c8530f8c, entries=150, sequenceid=196, filesize=11.9 K 2024-11-19T16:15:28,158 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1601148ee1f598abf2fc9cfff51f933e in 521ms, sequenceid=196, compaction requested=true 2024-11-19T16:15:28,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:28,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:28,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-19T16:15:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-19T16:15:28,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-19T16:15:28,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1360 sec 2024-11-19T16:15:28,166 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.1450 sec 2024-11-19T16:15:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:28,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T16:15:28,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:28,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:28,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:28,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:28,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:28,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:28,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/973180b9b7b848b09fa151bbdf578642 is 50, key is test_row_0/A:col10/1732032928229/Put/seqid=0 2024-11-19T16:15:28,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742101_1277 (size=12151) 2024-11-19T16:15:28,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/973180b9b7b848b09fa151bbdf578642 2024-11-19T16:15:28,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/4e09a85feec1420593f1f27660d9996f is 50, key is test_row_0/B:col10/1732032928229/Put/seqid=0 2024-11-19T16:15:28,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742102_1278 (size=12151) 2024-11-19T16:15:28,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/4e09a85feec1420593f1f27660d9996f 2024-11-19T16:15:28,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/4ffe8440e68948bfb0ef363655f6ed06 is 50, key is test_row_0/C:col10/1732032928229/Put/seqid=0 2024-11-19T16:15:28,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032988294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:28,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742103_1279 (size=12151) 2024-11-19T16:15:28,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/4ffe8440e68948bfb0ef363655f6ed06 2024-11-19T16:15:28,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/973180b9b7b848b09fa151bbdf578642 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/973180b9b7b848b09fa151bbdf578642 2024-11-19T16:15:28,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/973180b9b7b848b09fa151bbdf578642, entries=150, sequenceid=214, filesize=11.9 K 2024-11-19T16:15:28,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/4e09a85feec1420593f1f27660d9996f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4e09a85feec1420593f1f27660d9996f 2024-11-19T16:15:28,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4e09a85feec1420593f1f27660d9996f, entries=150, sequenceid=214, filesize=11.9 K 2024-11-19T16:15:28,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/4ffe8440e68948bfb0ef363655f6ed06 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/4ffe8440e68948bfb0ef363655f6ed06 2024-11-19T16:15:28,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/4ffe8440e68948bfb0ef363655f6ed06, entries=150, sequenceid=214, filesize=11.9 K 2024-11-19T16:15:28,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1601148ee1f598abf2fc9cfff51f933e in 112ms, sequenceid=214, compaction requested=true 2024-11-19T16:15:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:28,342 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T16:15:28,343 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:28,344 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51268 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:28,344 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/A is initiating minor compaction (all files) 2024-11-19T16:15:28,344 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/A in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:28,344 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/a99b48c8a8354efb806b9bd93f37c6b9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/30cfe8187f094756bb00c2f806eb343b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/117a37d48bf44e9289ce9254929dd797, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/973180b9b7b848b09fa151bbdf578642] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=50.1 K 2024-11-19T16:15:28,345 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:28,345 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/B is initiating minor compaction (all files) 2024-11-19T16:15:28,345 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/B in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:28,345 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/69821af53a534e16b12e38c72dffceca, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/5187643330584e9d969035483ce3c3b3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0764508a07864a5a8f5a21714a49bfe0, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4e09a85feec1420593f1f27660d9996f] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=47.7 K 2024-11-19T16:15:28,346 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting a99b48c8a8354efb806b9bd93f37c6b9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732032926004 2024-11-19T16:15:28,346 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 69821af53a534e16b12e38c72dffceca, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732032926004 2024-11-19T16:15:28,348 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 5187643330584e9d969035483ce3c3b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732032926994 2024-11-19T16:15:28,348 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30cfe8187f094756bb00c2f806eb343b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732032926994 2024-11-19T16:15:28,349 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 0764508a07864a5a8f5a21714a49bfe0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732032927187 2024-11-19T16:15:28,349 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 117a37d48bf44e9289ce9254929dd797, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732032927187 2024-11-19T16:15:28,349 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e09a85feec1420593f1f27660d9996f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732032927898 2024-11-19T16:15:28,349 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 973180b9b7b848b09fa151bbdf578642, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732032927898 2024-11-19T16:15:28,360 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:28,362 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/e5a106caa54d492580e538e9350e7242 is 50, key is test_row_0/B:col10/1732032928229/Put/seqid=0 2024-11-19T16:15:28,368 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#A#compaction#235 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:28,369 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/8fac6a1dafd4460ea566439ed1a154bf is 50, key is test_row_0/A:col10/1732032928229/Put/seqid=0 2024-11-19T16:15:28,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742104_1280 (size=12561) 2024-11-19T16:15:28,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742105_1281 (size=12561) 2024-11-19T16:15:28,393 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/8fac6a1dafd4460ea566439ed1a154bf as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/8fac6a1dafd4460ea566439ed1a154bf 2024-11-19T16:15:28,401 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/A of 1601148ee1f598abf2fc9cfff51f933e into 8fac6a1dafd4460ea566439ed1a154bf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:28,401 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:28,401 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/A, priority=12, startTime=1732032928342; duration=0sec 2024-11-19T16:15:28,401 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:28,401 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:A 2024-11-19T16:15:28,401 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:28,405 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:28,406 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/C is initiating minor compaction (all files) 2024-11-19T16:15:28,406 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/C in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:28,406 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/3c21b8fd8cc64d93847a1be8debb5f2e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/7e636a9cb5d9415ba5b0c2c136a2ac29, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/acbf4b171c5f47048a347830c8530f8c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/4ffe8440e68948bfb0ef363655f6ed06] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=47.7 K 2024-11-19T16:15:28,407 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c21b8fd8cc64d93847a1be8debb5f2e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732032926004 2024-11-19T16:15:28,407 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e636a9cb5d9415ba5b0c2c136a2ac29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732032926994 2024-11-19T16:15:28,408 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting acbf4b171c5f47048a347830c8530f8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732032927187 2024-11-19T16:15:28,408 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ffe8440e68948bfb0ef363655f6ed06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732032927898 2024-11-19T16:15:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:28,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T16:15:28,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:28,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:28,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:28,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:28,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:28,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:28,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/9ed68942b69b49ed938999f6a37b732c is 50, key is test_row_0/A:col10/1732032928292/Put/seqid=0 2024-11-19T16:15:28,442 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:28,463 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/3bd319dbb5354af9b41fe759bb4f2506 is 50, key is test_row_0/C:col10/1732032928229/Put/seqid=0 2024-11-19T16:15:28,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742106_1282 (size=14541) 2024-11-19T16:15:28,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/9ed68942b69b49ed938999f6a37b732c 2024-11-19T16:15:28,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/39afad95444f44c7aa8e7bfe98cd1340 is 50, key is test_row_0/B:col10/1732032928292/Put/seqid=0 2024-11-19T16:15:28,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742107_1283 (size=12561) 2024-11-19T16:15:28,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742108_1284 (size=12151) 2024-11-19T16:15:28,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/39afad95444f44c7aa8e7bfe98cd1340 2024-11-19T16:15:28,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/7310b89a0cdf409d924dfe1858a85854 is 50, key is test_row_0/C:col10/1732032928292/Put/seqid=0 2024-11-19T16:15:28,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742109_1285 (size=12151) 2024-11-19T16:15:28,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032988528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:28,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032988646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:28,774 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/e5a106caa54d492580e538e9350e7242 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/e5a106caa54d492580e538e9350e7242 2024-11-19T16:15:28,778 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/B of 1601148ee1f598abf2fc9cfff51f933e into e5a106caa54d492580e538e9350e7242(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:28,778 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:28,779 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/B, priority=12, startTime=1732032928342; duration=0sec 2024-11-19T16:15:28,779 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:28,779 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:B 2024-11-19T16:15:28,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032988857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:28,906 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/3bd319dbb5354af9b41fe759bb4f2506 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/3bd319dbb5354af9b41fe759bb4f2506 2024-11-19T16:15:28,913 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/C of 1601148ee1f598abf2fc9cfff51f933e into 3bd319dbb5354af9b41fe759bb4f2506(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:28,913 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:28,913 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/C, priority=12, startTime=1732032928342; duration=0sec 2024-11-19T16:15:28,913 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:28,913 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:C 2024-11-19T16:15:28,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/7310b89a0cdf409d924dfe1858a85854 2024-11-19T16:15:28,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/9ed68942b69b49ed938999f6a37b732c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9ed68942b69b49ed938999f6a37b732c 2024-11-19T16:15:28,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9ed68942b69b49ed938999f6a37b732c, entries=200, sequenceid=230, filesize=14.2 K 2024-11-19T16:15:28,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/39afad95444f44c7aa8e7bfe98cd1340 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/39afad95444f44c7aa8e7bfe98cd1340 2024-11-19T16:15:28,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/39afad95444f44c7aa8e7bfe98cd1340, entries=150, sequenceid=230, filesize=11.9 K 2024-11-19T16:15:28,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/7310b89a0cdf409d924dfe1858a85854 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/7310b89a0cdf409d924dfe1858a85854 2024-11-19T16:15:28,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/7310b89a0cdf409d924dfe1858a85854, entries=150, sequenceid=230, filesize=11.9 K 2024-11-19T16:15:28,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1601148ee1f598abf2fc9cfff51f933e in 559ms, sequenceid=230, compaction requested=false 2024-11-19T16:15:28,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:29,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T16:15:29,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:29,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:29,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:29,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:29,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:29,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:29,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/ab634cd18a3248279ee5194eb9afaaac is 50, key is test_row_0/A:col10/1732032928527/Put/seqid=0 2024-11-19T16:15:29,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032989161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032989163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032989167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032989169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032989169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T16:15:29,177 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:29,177 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-19T16:15:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T16:15:29,180 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:29,181 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:29,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:29,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742110_1286 (size=14541) 2024-11-19T16:15:29,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/ab634cd18a3248279ee5194eb9afaaac 2024-11-19T16:15:29,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/4d8948273f264da6809d2f076cabc801 is 50, key is test_row_0/B:col10/1732032928527/Put/seqid=0 2024-11-19T16:15:29,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742111_1287 (size=12151) 2024-11-19T16:15:29,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/4d8948273f264da6809d2f076cabc801 2024-11-19T16:15:29,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/953773a9d17b4faea4bc69ebfbf51579 is 50, key is test_row_0/C:col10/1732032928527/Put/seqid=0 2024-11-19T16:15:29,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742112_1288 (size=12151) 2024-11-19T16:15:29,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/953773a9d17b4faea4bc69ebfbf51579 2024-11-19T16:15:29,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/ab634cd18a3248279ee5194eb9afaaac as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/ab634cd18a3248279ee5194eb9afaaac 2024-11-19T16:15:29,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/ab634cd18a3248279ee5194eb9afaaac, entries=200, sequenceid=251, filesize=14.2 K 2024-11-19T16:15:29,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/4d8948273f264da6809d2f076cabc801 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4d8948273f264da6809d2f076cabc801 2024-11-19T16:15:29,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4d8948273f264da6809d2f076cabc801, entries=150, sequenceid=251, filesize=11.9 K 2024-11-19T16:15:29,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/953773a9d17b4faea4bc69ebfbf51579 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/953773a9d17b4faea4bc69ebfbf51579 2024-11-19T16:15:29,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/953773a9d17b4faea4bc69ebfbf51579, entries=150, sequenceid=251, filesize=11.9 K 2024-11-19T16:15:29,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1601148ee1f598abf2fc9cfff51f933e in 117ms, sequenceid=251, compaction requested=true 2024-11-19T16:15:29,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:29,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:29,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:29,266 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:29,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:29,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:29,266 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:29,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:29,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:29,267 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:29,267 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/A is initiating minor compaction (all files) 2024-11-19T16:15:29,267 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/A in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:29,268 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/8fac6a1dafd4460ea566439ed1a154bf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9ed68942b69b49ed938999f6a37b732c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/ab634cd18a3248279ee5194eb9afaaac] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=40.7 K 2024-11-19T16:15:29,268 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fac6a1dafd4460ea566439ed1a154bf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732032927898 2024-11-19T16:15:29,268 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:29,268 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/B is initiating minor compaction (all files) 2024-11-19T16:15:29,268 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/B in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:29,268 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ed68942b69b49ed938999f6a37b732c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732032928278 2024-11-19T16:15:29,268 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/e5a106caa54d492580e538e9350e7242, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/39afad95444f44c7aa8e7bfe98cd1340, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4d8948273f264da6809d2f076cabc801] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=36.0 K 2024-11-19T16:15:29,269 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab634cd18a3248279ee5194eb9afaaac, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732032928514 2024-11-19T16:15:29,269 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting e5a106caa54d492580e538e9350e7242, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732032927898 2024-11-19T16:15:29,270 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 39afad95444f44c7aa8e7bfe98cd1340, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732032928284 2024-11-19T16:15:29,270 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d8948273f264da6809d2f076cabc801, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732032928514 2024-11-19T16:15:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:29,282 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:29,282 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:29,282 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/3228ee679a2546398ebe6f07bd984b48 is 50, key is test_row_0/A:col10/1732032928527/Put/seqid=0 2024-11-19T16:15:29,282 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/0d363a4782eb4e079411fff0def44527 is 50, key is test_row_0/B:col10/1732032928527/Put/seqid=0 2024-11-19T16:15:29,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T16:15:29,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:29,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:29,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:29,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:29,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:29,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T16:15:29,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/de6fc450fe44405abe4832b6b6223f85 is 50, key is test_row_0/A:col10/1732032929167/Put/seqid=0 2024-11-19T16:15:29,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742113_1289 (size=12663) 2024-11-19T16:15:29,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742114_1290 (size=12663) 2024-11-19T16:15:29,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742115_1291 (size=12301) 2024-11-19T16:15:29,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032989311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032989312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032989313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032989315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T16:15:29,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:29,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:29,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:29,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:29,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:29,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032989426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032989426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032989427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032989427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T16:15:29,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:29,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:29,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:29,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:29,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T16:15:29,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032989637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032989637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032989637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032989637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T16:15:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:29,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:29,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:29,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:29,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032989686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:29,696 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/3228ee679a2546398ebe6f07bd984b48 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3228ee679a2546398ebe6f07bd984b48 2024-11-19T16:15:29,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/de6fc450fe44405abe4832b6b6223f85 2024-11-19T16:15:29,698 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/0d363a4782eb4e079411fff0def44527 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0d363a4782eb4e079411fff0def44527 2024-11-19T16:15:29,702 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/A of 1601148ee1f598abf2fc9cfff51f933e into 3228ee679a2546398ebe6f07bd984b48(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:29,702 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:29,702 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/A, priority=13, startTime=1732032929266; duration=0sec 2024-11-19T16:15:29,703 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:29,703 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:A 2024-11-19T16:15:29,703 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:29,704 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:29,704 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/C is initiating minor compaction (all files) 2024-11-19T16:15:29,704 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/C in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:29,704 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/3bd319dbb5354af9b41fe759bb4f2506, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/7310b89a0cdf409d924dfe1858a85854, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/953773a9d17b4faea4bc69ebfbf51579] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=36.0 K 2024-11-19T16:15:29,706 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bd319dbb5354af9b41fe759bb4f2506, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732032927898 2024-11-19T16:15:29,706 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/B of 1601148ee1f598abf2fc9cfff51f933e into 0d363a4782eb4e079411fff0def44527(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:29,706 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:29,706 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/B, priority=13, startTime=1732032929266; duration=0sec 2024-11-19T16:15:29,706 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7310b89a0cdf409d924dfe1858a85854, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732032928284 2024-11-19T16:15:29,706 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:29,706 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:B 2024-11-19T16:15:29,706 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 953773a9d17b4faea4bc69ebfbf51579, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732032928514 2024-11-19T16:15:29,713 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#C#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:29,713 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/2de20497b3614c6b9ba46c9ddb452009 is 50, key is test_row_0/C:col10/1732032928527/Put/seqid=0 2024-11-19T16:15:29,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/220d4fc1af084a1dbcbe713f0e75041c is 50, key is test_row_0/B:col10/1732032929167/Put/seqid=0 2024-11-19T16:15:29,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742116_1292 (size=12663) 2024-11-19T16:15:29,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742117_1293 (size=12301) 2024-11-19T16:15:29,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/220d4fc1af084a1dbcbe713f0e75041c 2024-11-19T16:15:29,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/169b13460c44449e87f5a65f1af46249 is 50, key is test_row_0/C:col10/1732032929167/Put/seqid=0 2024-11-19T16:15:29,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742118_1294 (size=12301) 2024-11-19T16:15:29,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/169b13460c44449e87f5a65f1af46249 2024-11-19T16:15:29,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/de6fc450fe44405abe4832b6b6223f85 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/de6fc450fe44405abe4832b6b6223f85 2024-11-19T16:15:29,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/de6fc450fe44405abe4832b6b6223f85, entries=150, sequenceid=267, filesize=12.0 K 2024-11-19T16:15:29,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/220d4fc1af084a1dbcbe713f0e75041c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/220d4fc1af084a1dbcbe713f0e75041c 2024-11-19T16:15:29,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/220d4fc1af084a1dbcbe713f0e75041c, entries=150, sequenceid=267, filesize=12.0 K 2024-11-19T16:15:29,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/169b13460c44449e87f5a65f1af46249 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/169b13460c44449e87f5a65f1af46249 2024-11-19T16:15:29,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/169b13460c44449e87f5a65f1af46249, entries=150, sequenceid=267, filesize=12.0 K 2024-11-19T16:15:29,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1601148ee1f598abf2fc9cfff51f933e in 483ms, sequenceid=267, compaction requested=false 2024-11-19T16:15:29,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:29,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T16:15:29,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:29,801 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T16:15:29,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:29,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:29,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:29,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:29,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:29,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:29,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/3d0eea2573b249eeba4428dc145ad52a is 50, key is test_row_0/A:col10/1732032929312/Put/seqid=0 2024-11-19T16:15:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T16:15:29,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742119_1295 (size=12301) 2024-11-19T16:15:29,818 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/3d0eea2573b249eeba4428dc145ad52a 2024-11-19T16:15:29,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/b03e8aebc8084c76b0db4ec6781f2fba is 50, key is test_row_0/B:col10/1732032929312/Put/seqid=0 2024-11-19T16:15:29,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742120_1296 (size=12301) 2024-11-19T16:15:29,891 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/b03e8aebc8084c76b0db4ec6781f2fba 2024-11-19T16:15:29,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/a45348b23bfd459286da3bcfece6a3b3 is 50, key is test_row_0/C:col10/1732032929312/Put/seqid=0 2024-11-19T16:15:29,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742121_1297 (size=12301) 2024-11-19T16:15:29,922 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/a45348b23bfd459286da3bcfece6a3b3 2024-11-19T16:15:29,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/3d0eea2573b249eeba4428dc145ad52a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3d0eea2573b249eeba4428dc145ad52a 2024-11-19T16:15:29,933 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3d0eea2573b249eeba4428dc145ad52a, entries=150, sequenceid=285, filesize=12.0 K 2024-11-19T16:15:29,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/b03e8aebc8084c76b0db4ec6781f2fba as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/b03e8aebc8084c76b0db4ec6781f2fba 2024-11-19T16:15:29,942 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/b03e8aebc8084c76b0db4ec6781f2fba, entries=150, sequenceid=285, filesize=12.0 K 2024-11-19T16:15:29,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/a45348b23bfd459286da3bcfece6a3b3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a45348b23bfd459286da3bcfece6a3b3 2024-11-19T16:15:29,948 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a45348b23bfd459286da3bcfece6a3b3, entries=150, sequenceid=285, filesize=12.0 K 2024-11-19T16:15:29,950 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 1601148ee1f598abf2fc9cfff51f933e in 149ms, sequenceid=285, compaction requested=true 2024-11-19T16:15:29,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:29,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:29,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-19T16:15:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-19T16:15:29,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-19T16:15:29,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 771 msec 2024-11-19T16:15:29,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 779 msec 2024-11-19T16:15:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:29,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T16:15:29,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:29,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:29,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:29,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:29,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:29,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:29,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/2976f248c4b74c92bdd76f4cc329abd8 is 50, key is test_row_1/A:col10/1732032929959/Put/seqid=0 2024-11-19T16:15:30,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742122_1298 (size=14737) 2024-11-19T16:15:30,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032929988, max timestamp=1732032930013Num uniques 859; 2024-11-19T16:15:30,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:15:30,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032929988, max timestamp=1732032930013Num uniques 859; 2024-11-19T16:15:30,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732032929996, max timestamp=1732032930013Num uniques 859; 2024-11-19T16:15:30,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T16:15:30,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:15:30,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032990050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032990051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032990054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032990054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,124 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/2de20497b3614c6b9ba46c9ddb452009 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2de20497b3614c6b9ba46c9ddb452009 2024-11-19T16:15:30,131 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/C of 1601148ee1f598abf2fc9cfff51f933e into 2de20497b3614c6b9ba46c9ddb452009(size=12.4 K), total size for store is 36.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:30,131 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:30,131 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/C, priority=13, startTime=1732032929266; duration=0sec 2024-11-19T16:15:30,132 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:30,132 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:C 2024-11-19T16:15:30,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032990167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032990167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032990167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032990167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T16:15:30,317 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:30,317 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-19T16:15:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T16:15:30,319 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:30,320 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:30,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:30,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032990376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032990376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032990376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032990376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/2976f248c4b74c92bdd76f4cc329abd8 2024-11-19T16:15:30,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/02c1880f914c4849bd8914cc1292bc4b is 50, key is test_row_1/B:col10/1732032929959/Put/seqid=0 2024-11-19T16:15:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T16:15:30,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742123_1299 (size=9857) 2024-11-19T16:15:30,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T16:15:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:30,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:30,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T16:15:30,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:30,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:30,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:30,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:30,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T16:15:30,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032990687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032990687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032990687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032990687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032990696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:30,701 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 1601148ee1f598abf2fc9cfff51f933e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:30,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:30,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:30,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T16:15:30,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:30,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:30,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:30,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:30,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:30,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/02c1880f914c4849bd8914cc1292bc4b 2024-11-19T16:15:30,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/cc66d1c5a4e94c9096f71314836a6891 is 50, key is test_row_1/C:col10/1732032929959/Put/seqid=0 2024-11-19T16:15:30,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742124_1300 (size=9857) 2024-11-19T16:15:30,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T16:15:30,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:30,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:30,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:30,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:30,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T16:15:31,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T16:15:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:31,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:31,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:31,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032991197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032991197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:31,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:31,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:31,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:31,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032991197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:31,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032991207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:31,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T16:15:31,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:31,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:31,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:31,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:31,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:31,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/cc66d1c5a4e94c9096f71314836a6891 2024-11-19T16:15:31,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/2976f248c4b74c92bdd76f4cc329abd8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/2976f248c4b74c92bdd76f4cc329abd8 2024-11-19T16:15:31,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/2976f248c4b74c92bdd76f4cc329abd8, entries=200, sequenceid=296, filesize=14.4 K 2024-11-19T16:15:31,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/02c1880f914c4849bd8914cc1292bc4b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/02c1880f914c4849bd8914cc1292bc4b 2024-11-19T16:15:31,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/02c1880f914c4849bd8914cc1292bc4b, entries=100, sequenceid=296, filesize=9.6 K 2024-11-19T16:15:31,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/cc66d1c5a4e94c9096f71314836a6891 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/cc66d1c5a4e94c9096f71314836a6891 2024-11-19T16:15:31,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/cc66d1c5a4e94c9096f71314836a6891, entries=100, sequenceid=296, filesize=9.6 K 2024-11-19T16:15:31,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.31 KB/132415 for 1601148ee1f598abf2fc9cfff51f933e in 1331ms, sequenceid=296, compaction requested=true 2024-11-19T16:15:31,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:31,291 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:31,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:31,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:31,292 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:31,292 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52002 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:31,292 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/A is initiating minor compaction (all files) 2024-11-19T16:15:31,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:31,293 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/A in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:31,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:31,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:31,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:31,293 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3228ee679a2546398ebe6f07bd984b48, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/de6fc450fe44405abe4832b6b6223f85, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3d0eea2573b249eeba4428dc145ad52a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/2976f248c4b74c92bdd76f4cc329abd8] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=50.8 K 2024-11-19T16:15:31,294 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3228ee679a2546398ebe6f07bd984b48, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732032928514 2024-11-19T16:15:31,295 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting de6fc450fe44405abe4832b6b6223f85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732032929159 2024-11-19T16:15:31,295 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47122 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:31,295 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/B is initiating minor compaction (all files) 2024-11-19T16:15:31,295 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/B in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:31,295 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0d363a4782eb4e079411fff0def44527, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/220d4fc1af084a1dbcbe713f0e75041c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/b03e8aebc8084c76b0db4ec6781f2fba, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/02c1880f914c4849bd8914cc1292bc4b] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=46.0 K 2024-11-19T16:15:31,297 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d0eea2573b249eeba4428dc145ad52a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732032929308 2024-11-19T16:15:31,298 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d363a4782eb4e079411fff0def44527, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732032928514 2024-11-19T16:15:31,298 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2976f248c4b74c92bdd76f4cc329abd8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732032929951 2024-11-19T16:15:31,298 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 220d4fc1af084a1dbcbe713f0e75041c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732032929159 2024-11-19T16:15:31,299 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting b03e8aebc8084c76b0db4ec6781f2fba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732032929308 2024-11-19T16:15:31,299 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c1880f914c4849bd8914cc1292bc4b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732032929958 2024-11-19T16:15:31,322 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:31,323 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/d37c383cbc824c76a9b3ab3a3c2408f1 is 50, key is test_row_0/B:col10/1732032929312/Put/seqid=0 2024-11-19T16:15:31,343 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#A#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:31,344 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/3497ead5010b48f6b6bd7a027f2f0572 is 50, key is test_row_0/A:col10/1732032929312/Put/seqid=0 2024-11-19T16:15:31,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742125_1301 (size=12949) 2024-11-19T16:15:31,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742126_1302 (size=12949) 2024-11-19T16:15:31,374 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/3497ead5010b48f6b6bd7a027f2f0572 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3497ead5010b48f6b6bd7a027f2f0572 2024-11-19T16:15:31,380 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/A of 1601148ee1f598abf2fc9cfff51f933e into 3497ead5010b48f6b6bd7a027f2f0572(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:31,380 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:31,380 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/A, priority=12, startTime=1732032931291; duration=0sec 2024-11-19T16:15:31,380 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:31,380 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:A 2024-11-19T16:15:31,381 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:31,381 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47122 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:31,381 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/C is initiating minor compaction (all files) 2024-11-19T16:15:31,382 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/C in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:31,382 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2de20497b3614c6b9ba46c9ddb452009, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/169b13460c44449e87f5a65f1af46249, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a45348b23bfd459286da3bcfece6a3b3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/cc66d1c5a4e94c9096f71314836a6891] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=46.0 K 2024-11-19T16:15:31,382 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2de20497b3614c6b9ba46c9ddb452009, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732032928514 2024-11-19T16:15:31,382 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 169b13460c44449e87f5a65f1af46249, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732032929159 2024-11-19T16:15:31,382 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting a45348b23bfd459286da3bcfece6a3b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732032929308 2024-11-19T16:15:31,383 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc66d1c5a4e94c9096f71314836a6891, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732032929958 2024-11-19T16:15:31,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T16:15:31,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:31,402 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=129.31 KB heapSize=364.54 KB 2024-11-19T16:15:31,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:31,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:31,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:31,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:31,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:31,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:31,408 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#C#compaction#257 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:31,409 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/a3b6f9ca00ea4e509f6ed00dc814693b is 50, key is test_row_0/C:col10/1732032929312/Put/seqid=0 2024-11-19T16:15:31,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/9b7ec585a58e4f61bb05718adeb34869 is 50, key is test_row_0/A:col10/1732032930049/Put/seqid=0 2024-11-19T16:15:31,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742127_1303 (size=12301) 2024-11-19T16:15:31,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742128_1304 (size=12949) 2024-11-19T16:15:31,418 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/9b7ec585a58e4f61bb05718adeb34869 2024-11-19T16:15:31,427 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/a3b6f9ca00ea4e509f6ed00dc814693b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a3b6f9ca00ea4e509f6ed00dc814693b 2024-11-19T16:15:31,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/7191d36e761049c0b9e89be34b32cc73 is 50, key is test_row_0/B:col10/1732032930049/Put/seqid=0 2024-11-19T16:15:31,434 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/C of 1601148ee1f598abf2fc9cfff51f933e into a3b6f9ca00ea4e509f6ed00dc814693b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:31,434 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:31,434 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/C, priority=12, startTime=1732032931293; duration=0sec 2024-11-19T16:15:31,434 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:31,434 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:C 2024-11-19T16:15:31,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742129_1305 (size=12301) 2024-11-19T16:15:31,436 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/7191d36e761049c0b9e89be34b32cc73 2024-11-19T16:15:31,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/0e3185bd07ab4ff79783fa357b5e2118 is 50, key is test_row_0/C:col10/1732032930049/Put/seqid=0 2024-11-19T16:15:31,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742130_1306 (size=12301) 2024-11-19T16:15:31,451 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/0e3185bd07ab4ff79783fa357b5e2118 2024-11-19T16:15:31,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/9b7ec585a58e4f61bb05718adeb34869 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9b7ec585a58e4f61bb05718adeb34869 2024-11-19T16:15:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T16:15:31,461 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9b7ec585a58e4f61bb05718adeb34869, entries=150, sequenceid=333, filesize=12.0 K 2024-11-19T16:15:31,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/7191d36e761049c0b9e89be34b32cc73 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/7191d36e761049c0b9e89be34b32cc73 2024-11-19T16:15:31,465 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/7191d36e761049c0b9e89be34b32cc73, entries=150, sequenceid=333, filesize=12.0 K 2024-11-19T16:15:31,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/0e3185bd07ab4ff79783fa357b5e2118 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0e3185bd07ab4ff79783fa357b5e2118 2024-11-19T16:15:31,469 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0e3185bd07ab4ff79783fa357b5e2118, entries=150, sequenceid=333, filesize=12.0 K 2024-11-19T16:15:31,470 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~129.31 KB/132415, heapSize ~364.49 KB/373240, currentSize=0 B/0 for 1601148ee1f598abf2fc9cfff51f933e in 68ms, sequenceid=333, compaction requested=false 2024-11-19T16:15:31,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:31,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:31,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-19T16:15:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-19T16:15:31,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-19T16:15:31,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1530 sec 2024-11-19T16:15:31,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.1610 sec 2024-11-19T16:15:31,763 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/d37c383cbc824c76a9b3ab3a3c2408f1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/d37c383cbc824c76a9b3ab3a3c2408f1 2024-11-19T16:15:31,768 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/B of 1601148ee1f598abf2fc9cfff51f933e into d37c383cbc824c76a9b3ab3a3c2408f1(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:31,768 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:31,768 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/B, priority=12, startTime=1732032931292; duration=0sec 2024-11-19T16:15:31,768 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:31,768 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:B 2024-11-19T16:15:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:32,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:15:32,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:32,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:32,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:32,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:32,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:32,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:32,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/d19a1f882d344dec81ab831baac2860b is 50, key is test_row_0/A:col10/1732032932213/Put/seqid=0 2024-11-19T16:15:32,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742131_1307 (size=14741) 2024-11-19T16:15:32,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032992239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032992241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032932230, max timestamp=1732032932243Num uniques 859; 2024-11-19T16:15:32,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032932230, max timestamp=1732032932243Num uniques 859; 2024-11-19T16:15:32,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032932230, max timestamp=1732032932243Num uniques 859; 2024-11-19T16:15:32,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:15:32,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:15:32,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:15:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032992282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032992285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032992357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032992357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032992397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032992397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T16:15:32,467 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:32,467 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-19T16:15:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T16:15:32,469 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:32,469 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:32,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:32,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032992566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032992566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T16:15:32,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032992607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032992607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T16:15:32,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:32,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:32,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:32,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:32,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:32,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/d19a1f882d344dec81ab831baac2860b 2024-11-19T16:15:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:32,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/39df1a9aea904514b2feb037383b74bc is 50, key is test_row_0/B:col10/1732032932213/Put/seqid=0 2024-11-19T16:15:32,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742132_1308 (size=12301) 2024-11-19T16:15:32,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032992727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,731 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 1601148ee1f598abf2fc9cfff51f933e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T16:15:32,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:32,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:32,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:32,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:32,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T16:15:32,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032992876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032992876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032992917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032992917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:32,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T16:15:32,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:32,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:32,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:32,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:32,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:33,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/39df1a9aea904514b2feb037383b74bc 2024-11-19T16:15:33,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/b142701af38c4a3ca245f6017f46e772 is 50, key is test_row_0/C:col10/1732032932213/Put/seqid=0 2024-11-19T16:15:33,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742133_1309 (size=12301) 2024-11-19T16:15:33,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T16:15:33,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:33,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:33,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:33,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:33,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T16:15:33,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T16:15:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:33,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:33,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:33,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T16:15:33,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:33,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:33,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:33,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:33,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:33,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:33,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032993387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:33,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:33,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:33,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:33,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:33,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032993387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:33,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032993427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:33,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:33,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032993427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:33,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/b142701af38c4a3ca245f6017f46e772 2024-11-19T16:15:33,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/d19a1f882d344dec81ab831baac2860b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/d19a1f882d344dec81ab831baac2860b 2024-11-19T16:15:33,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/d19a1f882d344dec81ab831baac2860b, entries=200, sequenceid=346, filesize=14.4 K 2024-11-19T16:15:33,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/39df1a9aea904514b2feb037383b74bc as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/39df1a9aea904514b2feb037383b74bc 2024-11-19T16:15:33,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/39df1a9aea904514b2feb037383b74bc, entries=150, sequenceid=346, filesize=12.0 K 2024-11-19T16:15:33,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/b142701af38c4a3ca245f6017f46e772 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b142701af38c4a3ca245f6017f46e772 2024-11-19T16:15:33,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b142701af38c4a3ca245f6017f46e772, entries=150, sequenceid=346, filesize=12.0 K 2024-11-19T16:15:33,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1601148ee1f598abf2fc9cfff51f933e in 1247ms, sequenceid=346, compaction requested=true 2024-11-19T16:15:33,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:33,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:33,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:33,462 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:33,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:33,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:33,462 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:33,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:33,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:33,462 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:33,462 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:33,463 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/B is initiating minor compaction (all files) 2024-11-19T16:15:33,463 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/A is initiating minor compaction (all files) 2024-11-19T16:15:33,463 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/B in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:33,463 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/A in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:33,463 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3497ead5010b48f6b6bd7a027f2f0572, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9b7ec585a58e4f61bb05718adeb34869, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/d19a1f882d344dec81ab831baac2860b] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=39.1 K 2024-11-19T16:15:33,463 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/d37c383cbc824c76a9b3ab3a3c2408f1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/7191d36e761049c0b9e89be34b32cc73, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/39df1a9aea904514b2feb037383b74bc] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=36.7 K 2024-11-19T16:15:33,463 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting d37c383cbc824c76a9b3ab3a3c2408f1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732032929312 2024-11-19T16:15:33,463 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3497ead5010b48f6b6bd7a027f2f0572, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732032929312 2024-11-19T16:15:33,463 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b7ec585a58e4f61bb05718adeb34869, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732032930049 2024-11-19T16:15:33,463 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 7191d36e761049c0b9e89be34b32cc73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732032930049 2024-11-19T16:15:33,464 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting d19a1f882d344dec81ab831baac2860b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732032932210 2024-11-19T16:15:33,464 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 39df1a9aea904514b2feb037383b74bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732032932210 2024-11-19T16:15:33,479 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#A#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:33,479 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#B#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:33,480 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/72a8bb2be2934070ab9fe7fc8bbbba23 is 50, key is test_row_0/B:col10/1732032932213/Put/seqid=0 2024-11-19T16:15:33,480 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/6674a5b5c97f452baae49c090a70a34a is 50, key is test_row_0/A:col10/1732032932213/Put/seqid=0 2024-11-19T16:15:33,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742134_1310 (size=13051) 2024-11-19T16:15:33,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742135_1311 (size=13051) 2024-11-19T16:15:33,491 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/6674a5b5c97f452baae49c090a70a34a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/6674a5b5c97f452baae49c090a70a34a 2024-11-19T16:15:33,491 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/72a8bb2be2934070ab9fe7fc8bbbba23 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/72a8bb2be2934070ab9fe7fc8bbbba23 2024-11-19T16:15:33,497 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/A of 1601148ee1f598abf2fc9cfff51f933e into 6674a5b5c97f452baae49c090a70a34a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:33,497 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/B of 1601148ee1f598abf2fc9cfff51f933e into 72a8bb2be2934070ab9fe7fc8bbbba23(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:33,497 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:33,497 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:33,497 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/B, priority=13, startTime=1732032933462; duration=0sec 2024-11-19T16:15:33,497 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/A, priority=13, startTime=1732032933461; duration=0sec 2024-11-19T16:15:33,497 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:33,497 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:A 2024-11-19T16:15:33,497 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:33,497 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:B 2024-11-19T16:15:33,497 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:33,499 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:33,499 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/C is initiating minor compaction (all files) 2024-11-19T16:15:33,499 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/C in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:33,499 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a3b6f9ca00ea4e509f6ed00dc814693b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0e3185bd07ab4ff79783fa357b5e2118, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b142701af38c4a3ca245f6017f46e772] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=36.7 K 2024-11-19T16:15:33,499 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3b6f9ca00ea4e509f6ed00dc814693b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732032929312 2024-11-19T16:15:33,499 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e3185bd07ab4ff79783fa357b5e2118, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732032930049 2024-11-19T16:15:33,500 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting b142701af38c4a3ca245f6017f46e772, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732032932210 2024-11-19T16:15:33,506 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#C#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:33,506 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/82b3114aebb0436784cb6e10d5f72d6c is 50, key is test_row_0/C:col10/1732032932213/Put/seqid=0 2024-11-19T16:15:33,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742136_1312 (size=13051) 2024-11-19T16:15:33,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T16:15:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:33,540 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-19T16:15:33,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:33,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:33,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:33,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:33,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:33,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:33,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/01f0975e995b4162ad7514de5a76864b is 50, key is test_row_0/A:col10/1732032932270/Put/seqid=0 2024-11-19T16:15:33,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742137_1313 (size=12301) 2024-11-19T16:15:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T16:15:33,914 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/82b3114aebb0436784cb6e10d5f72d6c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/82b3114aebb0436784cb6e10d5f72d6c 2024-11-19T16:15:33,918 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/C of 1601148ee1f598abf2fc9cfff51f933e into 82b3114aebb0436784cb6e10d5f72d6c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:33,918 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:33,918 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/C, priority=13, startTime=1732032933462; duration=0sec 2024-11-19T16:15:33,918 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:33,918 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:C 2024-11-19T16:15:33,948 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/01f0975e995b4162ad7514de5a76864b 2024-11-19T16:15:33,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/15d02d2c756d4a50a5db5dc55dfa92e7 is 50, key is test_row_0/B:col10/1732032932270/Put/seqid=0 2024-11-19T16:15:33,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742138_1314 (size=12301) 2024-11-19T16:15:33,974 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/15d02d2c756d4a50a5db5dc55dfa92e7 2024-11-19T16:15:33,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/b7951d978ef34e8c9d2f6220341936aa is 50, key is test_row_0/C:col10/1732032932270/Put/seqid=0 2024-11-19T16:15:33,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742139_1315 (size=12301) 2024-11-19T16:15:34,397 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/b7951d978ef34e8c9d2f6220341936aa 2024-11-19T16:15:34,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/01f0975e995b4162ad7514de5a76864b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/01f0975e995b4162ad7514de5a76864b 2024-11-19T16:15:34,408 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/01f0975e995b4162ad7514de5a76864b, entries=150, sequenceid=384, filesize=12.0 K 2024-11-19T16:15:34,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/15d02d2c756d4a50a5db5dc55dfa92e7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/15d02d2c756d4a50a5db5dc55dfa92e7 2024-11-19T16:15:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:34,416 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/15d02d2c756d4a50a5db5dc55dfa92e7, entries=150, sequenceid=384, filesize=12.0 K 2024-11-19T16:15:34,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/b7951d978ef34e8c9d2f6220341936aa as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b7951d978ef34e8c9d2f6220341936aa 2024-11-19T16:15:34,425 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b7951d978ef34e8c9d2f6220341936aa, entries=150, sequenceid=384, filesize=12.0 K 2024-11-19T16:15:34,426 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=33.54 KB/34350 for 1601148ee1f598abf2fc9cfff51f933e in 887ms, sequenceid=384, compaction requested=false 2024-11-19T16:15:34,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:34,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:34,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-19T16:15:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-19T16:15:34,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:34,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:15:34,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:34,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:34,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:34,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:34,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:34,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:34,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-19T16:15:34,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9580 sec 2024-11-19T16:15:34,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/0fceae7eee704e54855bf64329a667d9 is 50, key is test_row_0/A:col10/1732032934421/Put/seqid=0 2024-11-19T16:15:34,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.9640 sec 2024-11-19T16:15:34,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742140_1316 (size=14741) 2024-11-19T16:15:34,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032934483, max timestamp=1732032934488Num uniques 859; 2024-11-19T16:15:34,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732032934483, max timestamp=1732032934488Num uniques 859; 2024-11-19T16:15:34,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732032934483, max timestamp=1732032934488Num uniques 859; 2024-11-19T16:15:34,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:34,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:34,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:34,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032994511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:34,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:34,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032994542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:34,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032994542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032994542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T16:15:34,617 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:34,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-19T16:15:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T16:15:34,619 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:34,619 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:34,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:34,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032994649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:34,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032994657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:34,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032994657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:34,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032994657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:34,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T16:15:34,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T16:15:34,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:34,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:34,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/0fceae7eee704e54855bf64329a667d9 2024-11-19T16:15:34,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/fdc01d16a93d467d9b39ca092007a497 is 50, key is test_row_0/B:col10/1732032934421/Put/seqid=0 2024-11-19T16:15:34,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742141_1317 (size=12301) 2024-11-19T16:15:34,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032994867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:34,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:34,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032994877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:34,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032994877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:34,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032994877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:34,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T16:15:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:34,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T16:15:35,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T16:15:35,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:35,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:35,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:35,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:35,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:35,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032995177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:35,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032995187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:35,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032995187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:35,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032995187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:35,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T16:15:35,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:35,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:35,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:35,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:35,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T16:15:35,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/fdc01d16a93d467d9b39ca092007a497 2024-11-19T16:15:35,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/bc7a07c16dd44035bc025d9892ed9604 is 50, key is test_row_0/C:col10/1732032934421/Put/seqid=0 2024-11-19T16:15:35,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742142_1318 (size=12301) 2024-11-19T16:15:35,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/bc7a07c16dd44035bc025d9892ed9604 2024-11-19T16:15:35,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/0fceae7eee704e54855bf64329a667d9 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/0fceae7eee704e54855bf64329a667d9 2024-11-19T16:15:35,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/0fceae7eee704e54855bf64329a667d9, entries=200, sequenceid=395, filesize=14.4 K 2024-11-19T16:15:35,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/fdc01d16a93d467d9b39ca092007a497 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/fdc01d16a93d467d9b39ca092007a497 2024-11-19T16:15:35,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/fdc01d16a93d467d9b39ca092007a497, entries=150, sequenceid=395, filesize=12.0 K 2024-11-19T16:15:35,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/bc7a07c16dd44035bc025d9892ed9604 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/bc7a07c16dd44035bc025d9892ed9604 2024-11-19T16:15:35,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/bc7a07c16dd44035bc025d9892ed9604, entries=150, sequenceid=395, filesize=12.0 K 2024-11-19T16:15:35,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1601148ee1f598abf2fc9cfff51f933e in 870ms, sequenceid=395, compaction requested=true 2024-11-19T16:15:35,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:35,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:35,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:35,300 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:35,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:35,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:35,300 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:35,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:35,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:35,301 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:35,301 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:35,301 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/A is initiating minor compaction (all files) 2024-11-19T16:15:35,301 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/B is initiating minor compaction (all files) 2024-11-19T16:15:35,301 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/B in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:35,301 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/A in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:35,301 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/6674a5b5c97f452baae49c090a70a34a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/01f0975e995b4162ad7514de5a76864b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/0fceae7eee704e54855bf64329a667d9] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=39.2 K 2024-11-19T16:15:35,301 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/72a8bb2be2934070ab9fe7fc8bbbba23, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/15d02d2c756d4a50a5db5dc55dfa92e7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/fdc01d16a93d467d9b39ca092007a497] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=36.8 K 2024-11-19T16:15:35,301 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6674a5b5c97f452baae49c090a70a34a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732032932210 2024-11-19T16:15:35,301 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 72a8bb2be2934070ab9fe7fc8bbbba23, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732032932210 2024-11-19T16:15:35,302 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 15d02d2c756d4a50a5db5dc55dfa92e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732032932270 2024-11-19T16:15:35,302 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01f0975e995b4162ad7514de5a76864b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732032932270 2024-11-19T16:15:35,303 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting fdc01d16a93d467d9b39ca092007a497, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732032934409 2024-11-19T16:15:35,303 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fceae7eee704e54855bf64329a667d9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732032934409 2024-11-19T16:15:35,311 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#A#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:35,312 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/66d8e7317ac940af8a486521c0e9b0e1 is 50, key is test_row_0/A:col10/1732032934421/Put/seqid=0 2024-11-19T16:15:35,313 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#B#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:35,314 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/99b16b61068248d4a6bd73710430d7fa is 50, key is test_row_0/B:col10/1732032934421/Put/seqid=0 2024-11-19T16:15:35,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742144_1320 (size=13153) 2024-11-19T16:15:35,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742143_1319 (size=13153) 2024-11-19T16:15:35,341 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/99b16b61068248d4a6bd73710430d7fa as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/99b16b61068248d4a6bd73710430d7fa 2024-11-19T16:15:35,345 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/66d8e7317ac940af8a486521c0e9b0e1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/66d8e7317ac940af8a486521c0e9b0e1 2024-11-19T16:15:35,350 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/B of 1601148ee1f598abf2fc9cfff51f933e into 99b16b61068248d4a6bd73710430d7fa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:35,350 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:35,350 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/B, priority=13, startTime=1732032935300; duration=0sec 2024-11-19T16:15:35,350 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:35,350 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:B 2024-11-19T16:15:35,350 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:35,351 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:35,351 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/C is initiating minor compaction (all files) 2024-11-19T16:15:35,351 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/A of 1601148ee1f598abf2fc9cfff51f933e into 66d8e7317ac940af8a486521c0e9b0e1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:35,351 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:35,351 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/C in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:35,351 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/A, priority=13, startTime=1732032935299; duration=0sec 2024-11-19T16:15:35,351 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/82b3114aebb0436784cb6e10d5f72d6c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b7951d978ef34e8c9d2f6220341936aa, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/bc7a07c16dd44035bc025d9892ed9604] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=36.8 K 2024-11-19T16:15:35,351 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:35,351 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:A 2024-11-19T16:15:35,353 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 82b3114aebb0436784cb6e10d5f72d6c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732032932210 2024-11-19T16:15:35,353 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting b7951d978ef34e8c9d2f6220341936aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732032932270 2024-11-19T16:15:35,353 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting bc7a07c16dd44035bc025d9892ed9604, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732032934409 2024-11-19T16:15:35,360 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#C#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:35,360 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/ab0b9c5ad7284362becfa6c3bc49d217 is 50, key is test_row_0/C:col10/1732032934421/Put/seqid=0 2024-11-19T16:15:35,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T16:15:35,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:35,385 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T16:15:35,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:35,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:35,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:35,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:35,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:35,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:35,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742145_1321 (size=13153) 2024-11-19T16:15:35,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/cf7659c035a04097ae9625017242a448 is 50, key is test_row_0/A:col10/1732032934516/Put/seqid=0 2024-11-19T16:15:35,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742146_1322 (size=12301) 2024-11-19T16:15:35,425 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/cf7659c035a04097ae9625017242a448 2024-11-19T16:15:35,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/a8b5722639654c78ad94d5f7ac72e76b is 50, key is test_row_0/B:col10/1732032934516/Put/seqid=0 2024-11-19T16:15:35,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742147_1323 (size=12301) 2024-11-19T16:15:35,456 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/a8b5722639654c78ad94d5f7ac72e76b 2024-11-19T16:15:35,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/a33155433c39450182a2b70915f4771c is 50, key is test_row_0/C:col10/1732032934516/Put/seqid=0 2024-11-19T16:15:35,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742148_1324 (size=12301) 2024-11-19T16:15:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:35,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:35,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032995700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:35,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032995701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:35,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032995702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:35,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032995703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T16:15:35,795 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/ab0b9c5ad7284362becfa6c3bc49d217 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/ab0b9c5ad7284362becfa6c3bc49d217 2024-11-19T16:15:35,800 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/C of 1601148ee1f598abf2fc9cfff51f933e into ab0b9c5ad7284362becfa6c3bc49d217(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:35,800 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:35,800 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/C, priority=13, startTime=1732032935300; duration=0sec 2024-11-19T16:15:35,800 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:35,800 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:C 2024-11-19T16:15:35,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032995807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:35,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:35,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032995807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:35,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:35,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032995817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:35,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032995817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:35,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:35,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:35,927 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/a33155433c39450182a2b70915f4771c 2024-11-19T16:15:35,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/cf7659c035a04097ae9625017242a448 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/cf7659c035a04097ae9625017242a448 2024-11-19T16:15:35,935 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/cf7659c035a04097ae9625017242a448, entries=150, sequenceid=433, filesize=12.0 K 2024-11-19T16:15:35,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/a8b5722639654c78ad94d5f7ac72e76b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/a8b5722639654c78ad94d5f7ac72e76b 2024-11-19T16:15:35,940 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/a8b5722639654c78ad94d5f7ac72e76b, entries=150, sequenceid=433, filesize=12.0 K 2024-11-19T16:15:35,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/a33155433c39450182a2b70915f4771c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a33155433c39450182a2b70915f4771c 2024-11-19T16:15:35,946 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a33155433c39450182a2b70915f4771c, entries=150, sequenceid=433, filesize=12.0 K 2024-11-19T16:15:35,947 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 1601148ee1f598abf2fc9cfff51f933e in 562ms, sequenceid=433, compaction requested=false 2024-11-19T16:15:35,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:35,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:35,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-19T16:15:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-19T16:15:35,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-19T16:15:35,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3290 sec 2024-11-19T16:15:35,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3340 sec 2024-11-19T16:15:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:36,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T16:15:36,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:36,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:36,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:36,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:36,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:36,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:36,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/d6b13c69c84b4dcf883e8d1eeaca07ea is 50, key is test_row_0/A:col10/1732032935701/Put/seqid=0 2024-11-19T16:15:36,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742149_1325 (size=12301) 2024-11-19T16:15:36,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032996047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032936028, max timestamp=1732032936054Num uniques 859; 2024-11-19T16:15:36,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032936028, max timestamp=1732032936054Num uniques 859; 2024-11-19T16:15:36,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732032936028, max timestamp=1732032936054Num uniques 859; 2024-11-19T16:15:36,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:15:36,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:15:36,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:15:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032996084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032996087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032996089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032996167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032996196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032996196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032996207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032996377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032996407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032996407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032996416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/d6b13c69c84b4dcf883e8d1eeaca07ea 2024-11-19T16:15:36,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/fc3aa9c02f9a434b9d4a51e0cb25c758 is 50, key is test_row_0/B:col10/1732032935701/Put/seqid=0 2024-11-19T16:15:36,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742150_1326 (size=12301) 2024-11-19T16:15:36,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032996687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032996717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032996717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032996727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42570 deadline: 1732032996757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:36,761 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 1601148ee1f598abf2fc9cfff51f933e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T16:15:36,767 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:36,768 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-19T16:15:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T16:15:36,770 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:36,772 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:36,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:36,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/fc3aa9c02f9a434b9d4a51e0cb25c758 2024-11-19T16:15:36,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/51d801acf1ba415e9fa5b8fc04cd55c4 is 50, key is test_row_0/C:col10/1732032935701/Put/seqid=0 2024-11-19T16:15:36,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742151_1327 (size=12301) 2024-11-19T16:15:36,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T16:15:36,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T16:15:36,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:36,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:36,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:36,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:36,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:37,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T16:15:37,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:37,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:37,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:37,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:37,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T16:15:37,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032997196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:37,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032997227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:37,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032997227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:37,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T16:15:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:37,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:37,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:37,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:37,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:37,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:37,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032997237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:37,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/51d801acf1ba415e9fa5b8fc04cd55c4 2024-11-19T16:15:37,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/d6b13c69c84b4dcf883e8d1eeaca07ea as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/d6b13c69c84b4dcf883e8d1eeaca07ea 2024-11-19T16:15:37,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/d6b13c69c84b4dcf883e8d1eeaca07ea, entries=150, sequenceid=445, filesize=12.0 K 2024-11-19T16:15:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/fc3aa9c02f9a434b9d4a51e0cb25c758 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/fc3aa9c02f9a434b9d4a51e0cb25c758 2024-11-19T16:15:37,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/fc3aa9c02f9a434b9d4a51e0cb25c758, entries=150, sequenceid=445, filesize=12.0 K 2024-11-19T16:15:37,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/51d801acf1ba415e9fa5b8fc04cd55c4 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/51d801acf1ba415e9fa5b8fc04cd55c4 2024-11-19T16:15:37,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/51d801acf1ba415e9fa5b8fc04cd55c4, entries=150, sequenceid=445, filesize=12.0 K 2024-11-19T16:15:37,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1601148ee1f598abf2fc9cfff51f933e in 1258ms, sequenceid=445, compaction requested=true 2024-11-19T16:15:37,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:37,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:37,278 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:37,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:37,278 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:37,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:37,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:37,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:37,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:37,278 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:37,278 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:37,278 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/B is initiating minor compaction (all files) 2024-11-19T16:15:37,278 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/A is initiating minor compaction (all files) 2024-11-19T16:15:37,279 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/A in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:37,279 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/B in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:37,279 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/99b16b61068248d4a6bd73710430d7fa, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/a8b5722639654c78ad94d5f7ac72e76b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/fc3aa9c02f9a434b9d4a51e0cb25c758] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=36.9 K 2024-11-19T16:15:37,279 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/66d8e7317ac940af8a486521c0e9b0e1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/cf7659c035a04097ae9625017242a448, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/d6b13c69c84b4dcf883e8d1eeaca07ea] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=36.9 K 2024-11-19T16:15:37,279 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 99b16b61068248d4a6bd73710430d7fa, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732032934409 2024-11-19T16:15:37,279 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66d8e7317ac940af8a486521c0e9b0e1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732032934409 2024-11-19T16:15:37,279 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf7659c035a04097ae9625017242a448, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732032934497 2024-11-19T16:15:37,279 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a8b5722639654c78ad94d5f7ac72e76b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732032934497 2024-11-19T16:15:37,279 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6b13c69c84b4dcf883e8d1eeaca07ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732032935701 2024-11-19T16:15:37,279 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting fc3aa9c02f9a434b9d4a51e0cb25c758, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732032935701 2024-11-19T16:15:37,285 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#A#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:37,286 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#B#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:37,286 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/33551b38d589444fb9c04ed4a5ccebb1 is 50, key is test_row_0/A:col10/1732032935701/Put/seqid=0 2024-11-19T16:15:37,286 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/0161b046cfbc4e148f7769e1744d6bd4 is 50, key is test_row_0/B:col10/1732032935701/Put/seqid=0 2024-11-19T16:15:37,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742152_1328 (size=13255) 2024-11-19T16:15:37,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742153_1329 (size=13255) 2024-11-19T16:15:37,304 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/0161b046cfbc4e148f7769e1744d6bd4 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0161b046cfbc4e148f7769e1744d6bd4 2024-11-19T16:15:37,308 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/B of 1601148ee1f598abf2fc9cfff51f933e into 0161b046cfbc4e148f7769e1744d6bd4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:37,308 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:37,308 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/B, priority=13, startTime=1732032937278; duration=0sec 2024-11-19T16:15:37,308 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:37,308 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:B 2024-11-19T16:15:37,308 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:37,309 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:37,309 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/C is initiating minor compaction (all files) 2024-11-19T16:15:37,309 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/C in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:37,309 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/ab0b9c5ad7284362becfa6c3bc49d217, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a33155433c39450182a2b70915f4771c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/51d801acf1ba415e9fa5b8fc04cd55c4] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=36.9 K 2024-11-19T16:15:37,310 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting ab0b9c5ad7284362becfa6c3bc49d217, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732032934409 2024-11-19T16:15:37,310 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a33155433c39450182a2b70915f4771c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732032934497 2024-11-19T16:15:37,310 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 51d801acf1ba415e9fa5b8fc04cd55c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732032935701 2024-11-19T16:15:37,316 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#C#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:37,317 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/e800baa292e5423283133d69b3f37ad4 is 50, key is test_row_0/C:col10/1732032935701/Put/seqid=0 2024-11-19T16:15:37,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742154_1330 (size=13255) 2024-11-19T16:15:37,327 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/e800baa292e5423283133d69b3f37ad4 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/e800baa292e5423283133d69b3f37ad4 2024-11-19T16:15:37,333 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/C of 1601148ee1f598abf2fc9cfff51f933e into e800baa292e5423283133d69b3f37ad4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:37,333 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:37,333 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/C, priority=13, startTime=1732032937278; duration=0sec 2024-11-19T16:15:37,333 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:37,333 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:C 2024-11-19T16:15:37,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T16:15:37,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:37,382 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-19T16:15:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:37,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/3908fdfbb0284c13af38ef8a5553c639 is 50, key is test_row_0/A:col10/1732032936088/Put/seqid=0 2024-11-19T16:15:37,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742155_1331 (size=12301) 2024-11-19T16:15:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T16:15:37,699 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/33551b38d589444fb9c04ed4a5ccebb1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/33551b38d589444fb9c04ed4a5ccebb1 2024-11-19T16:15:37,703 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/A of 1601148ee1f598abf2fc9cfff51f933e into 33551b38d589444fb9c04ed4a5ccebb1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:37,703 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:37,703 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/A, priority=13, startTime=1732032937277; duration=0sec 2024-11-19T16:15:37,704 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:37,704 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:A 2024-11-19T16:15:37,792 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/3908fdfbb0284c13af38ef8a5553c639 2024-11-19T16:15:37,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/29e390e35479441f8893c4506f84c820 is 50, key is test_row_0/B:col10/1732032936088/Put/seqid=0 2024-11-19T16:15:37,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742156_1332 (size=12301) 2024-11-19T16:15:37,803 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/29e390e35479441f8893c4506f84c820 2024-11-19T16:15:37,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/41a0bc9924db4fdc819dcc673061ff4d is 50, key is test_row_0/C:col10/1732032936088/Put/seqid=0 2024-11-19T16:15:37,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742157_1333 (size=12301) 2024-11-19T16:15:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T16:15:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:38,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:38,217 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/41a0bc9924db4fdc819dcc673061ff4d 2024-11-19T16:15:38,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/3908fdfbb0284c13af38ef8a5553c639 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3908fdfbb0284c13af38ef8a5553c639 2024-11-19T16:15:38,224 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3908fdfbb0284c13af38ef8a5553c639, entries=150, sequenceid=481, filesize=12.0 K 2024-11-19T16:15:38,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/29e390e35479441f8893c4506f84c820 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/29e390e35479441f8893c4506f84c820 2024-11-19T16:15:38,228 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/29e390e35479441f8893c4506f84c820, entries=150, sequenceid=481, filesize=12.0 K 2024-11-19T16:15:38,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/41a0bc9924db4fdc819dcc673061ff4d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/41a0bc9924db4fdc819dcc673061ff4d 2024-11-19T16:15:38,232 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/41a0bc9924db4fdc819dcc673061ff4d, entries=150, sequenceid=481, filesize=12.0 K 2024-11-19T16:15:38,233 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=40.25 KB/41220 for 1601148ee1f598abf2fc9cfff51f933e in 851ms, sequenceid=481, compaction requested=false 2024-11-19T16:15:38,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:38,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:38,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-19T16:15:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-19T16:15:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:38,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:15:38,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:38,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:38,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:38,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:38,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:38,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:38,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-19T16:15:38,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4630 sec 2024-11-19T16:15:38,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4700 sec 2024-11-19T16:15:38,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/1f35b309e4fd408486cc9091dd600451 is 50, key is test_row_1/A:col10/1732032938232/Put/seqid=0 2024-11-19T16:15:38,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742158_1334 (size=12297) 2024-11-19T16:15:38,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032938260, max timestamp=1732032938268Num uniques 859; 2024-11-19T16:15:38,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:38,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032938260, max timestamp=1732032938268Num uniques 859; 2024-11-19T16:15:38,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032938260, max timestamp=1732032938268Num uniques 859; 2024-11-19T16:15:38,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:38,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:38,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032998288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032998289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032998290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032998293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032998397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032998397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032998406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032998406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032998607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032998607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032998616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032998616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/1f35b309e4fd408486cc9091dd600451 2024-11-19T16:15:38,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/640d85f2957e4a27ad3fc50ed9f8fb9d is 50, key is test_row_1/B:col10/1732032938232/Put/seqid=0 2024-11-19T16:15:38,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742159_1335 (size=9857) 2024-11-19T16:15:38,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032998917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032998917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T16:15:38,927 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:38,927 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-19T16:15:38,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T16:15:38,929 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:38,930 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:38,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:38,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032998926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032998927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T16:15:39,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/640d85f2957e4a27ad3fc50ed9f8fb9d 2024-11-19T16:15:39,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/0d0db7f382a24c19a32b7115e8a3db1f is 50, key is test_row_1/C:col10/1732032938232/Put/seqid=0 2024-11-19T16:15:39,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742160_1336 (size=9857) 2024-11-19T16:15:39,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T16:15:39,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:39,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:39,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:39,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:39,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:39,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T16:15:39,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:39,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:39,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:39,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:39,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T16:15:39,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T16:15:39,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:39,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:39,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:39,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:39,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:39,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732032999427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:39,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732032999427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:39,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:39,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732032999437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732032999437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:39,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/0d0db7f382a24c19a32b7115e8a3db1f 2024-11-19T16:15:39,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/1f35b309e4fd408486cc9091dd600451 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/1f35b309e4fd408486cc9091dd600451 2024-11-19T16:15:39,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/1f35b309e4fd408486cc9091dd600451, entries=150, sequenceid=492, filesize=12.0 K 2024-11-19T16:15:39,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/640d85f2957e4a27ad3fc50ed9f8fb9d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/640d85f2957e4a27ad3fc50ed9f8fb9d 2024-11-19T16:15:39,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/640d85f2957e4a27ad3fc50ed9f8fb9d, entries=100, sequenceid=492, filesize=9.6 K 2024-11-19T16:15:39,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/0d0db7f382a24c19a32b7115e8a3db1f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0d0db7f382a24c19a32b7115e8a3db1f 2024-11-19T16:15:39,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0d0db7f382a24c19a32b7115e8a3db1f, entries=100, sequenceid=492, filesize=9.6 K 2024-11-19T16:15:39,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1601148ee1f598abf2fc9cfff51f933e in 1277ms, sequenceid=492, compaction requested=true 2024-11-19T16:15:39,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:39,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:39,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:39,514 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:39,514 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:39,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:39,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:39,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:39,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:39,515 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37853 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:39,515 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/A is initiating minor compaction (all files) 2024-11-19T16:15:39,515 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/A in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:39,515 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/33551b38d589444fb9c04ed4a5ccebb1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3908fdfbb0284c13af38ef8a5553c639, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/1f35b309e4fd408486cc9091dd600451] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=37.0 K 2024-11-19T16:15:39,516 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:39,516 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/B is initiating minor compaction (all files) 2024-11-19T16:15:39,516 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/B in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:39,516 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0161b046cfbc4e148f7769e1744d6bd4, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/29e390e35479441f8893c4506f84c820, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/640d85f2957e4a27ad3fc50ed9f8fb9d] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=34.6 K 2024-11-19T16:15:39,516 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33551b38d589444fb9c04ed4a5ccebb1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732032935701 2024-11-19T16:15:39,517 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 0161b046cfbc4e148f7769e1744d6bd4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732032935701 2024-11-19T16:15:39,517 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 29e390e35479441f8893c4506f84c820, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732032936081 2024-11-19T16:15:39,517 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3908fdfbb0284c13af38ef8a5553c639, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732032936081 2024-11-19T16:15:39,518 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 640d85f2957e4a27ad3fc50ed9f8fb9d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1732032938232 2024-11-19T16:15:39,518 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f35b309e4fd408486cc9091dd600451, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1732032938223 2024-11-19T16:15:39,529 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#B#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:39,530 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#A#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:39,530 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/aeff524ad5cc4b23afe9b48147c79fb8 is 50, key is test_row_0/A:col10/1732032936088/Put/seqid=0 2024-11-19T16:15:39,532 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/f1930c0a465c47579dcd355e4edca8f5 is 50, key is test_row_0/B:col10/1732032936088/Put/seqid=0 2024-11-19T16:15:39,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T16:15:39,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:39,542 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T16:15:39,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:39,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:39,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:39,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:39,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:39,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T16:15:39,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742161_1337 (size=13357) 2024-11-19T16:15:39,577 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/aeff524ad5cc4b23afe9b48147c79fb8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/aeff524ad5cc4b23afe9b48147c79fb8 2024-11-19T16:15:39,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/0e96acbb92fc48b5a4f71d1729a2be53 is 50, key is test_row_0/A:col10/1732032938289/Put/seqid=0 2024-11-19T16:15:39,586 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/A of 1601148ee1f598abf2fc9cfff51f933e into aeff524ad5cc4b23afe9b48147c79fb8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:39,586 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:39,586 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/A, priority=13, startTime=1732032939513; duration=0sec 2024-11-19T16:15:39,586 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:39,587 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:A 2024-11-19T16:15:39,587 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:39,588 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:39,588 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/C is initiating minor compaction (all files) 2024-11-19T16:15:39,588 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/C in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:39,589 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/e800baa292e5423283133d69b3f37ad4, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/41a0bc9924db4fdc819dcc673061ff4d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0d0db7f382a24c19a32b7115e8a3db1f] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=34.6 K 2024-11-19T16:15:39,589 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting e800baa292e5423283133d69b3f37ad4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732032935701 2024-11-19T16:15:39,589 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41a0bc9924db4fdc819dcc673061ff4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732032936081 2024-11-19T16:15:39,589 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d0db7f382a24c19a32b7115e8a3db1f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1732032938232 2024-11-19T16:15:39,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742162_1338 (size=13357) 2024-11-19T16:15:39,621 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/f1930c0a465c47579dcd355e4edca8f5 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/f1930c0a465c47579dcd355e4edca8f5 2024-11-19T16:15:39,630 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/B of 1601148ee1f598abf2fc9cfff51f933e into f1930c0a465c47579dcd355e4edca8f5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:39,630 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:39,630 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/B, priority=13, startTime=1732032939514; duration=0sec 2024-11-19T16:15:39,630 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:39,630 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:B 2024-11-19T16:15:39,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742163_1339 (size=12301) 2024-11-19T16:15:39,636 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#C#compaction#294 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:39,636 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/da85cc23e45c4a3dba01b63223f66adf is 50, key is test_row_0/C:col10/1732032936088/Put/seqid=0 2024-11-19T16:15:39,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742164_1340 (size=13357) 2024-11-19T16:15:39,695 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/da85cc23e45c4a3dba01b63223f66adf as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/da85cc23e45c4a3dba01b63223f66adf 2024-11-19T16:15:39,709 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/C of 1601148ee1f598abf2fc9cfff51f933e into da85cc23e45c4a3dba01b63223f66adf(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:39,709 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:39,709 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/C, priority=13, startTime=1732032939515; duration=0sec 2024-11-19T16:15:39,709 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:39,709 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:C 2024-11-19T16:15:40,032 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/0e96acbb92fc48b5a4f71d1729a2be53 2024-11-19T16:15:40,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/7565d077ddc44fb9baf1b3050f29c057 is 50, key is test_row_0/B:col10/1732032938289/Put/seqid=0 2024-11-19T16:15:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T16:15:40,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742165_1341 (size=12301) 2024-11-19T16:15:40,087 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/7565d077ddc44fb9baf1b3050f29c057 2024-11-19T16:15:40,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/0d6278f2d6904a77afe0e92bbe04e7d5 is 50, key is test_row_0/C:col10/1732032938289/Put/seqid=0 2024-11-19T16:15:40,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742166_1342 (size=12301) 2024-11-19T16:15:40,124 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/0d6278f2d6904a77afe0e92bbe04e7d5 2024-11-19T16:15:40,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/0e96acbb92fc48b5a4f71d1729a2be53 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/0e96acbb92fc48b5a4f71d1729a2be53 2024-11-19T16:15:40,136 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/0e96acbb92fc48b5a4f71d1729a2be53, entries=150, sequenceid=528, filesize=12.0 K 2024-11-19T16:15:40,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/7565d077ddc44fb9baf1b3050f29c057 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/7565d077ddc44fb9baf1b3050f29c057 2024-11-19T16:15:40,146 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/7565d077ddc44fb9baf1b3050f29c057, entries=150, sequenceid=528, filesize=12.0 K 2024-11-19T16:15:40,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/0d6278f2d6904a77afe0e92bbe04e7d5 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0d6278f2d6904a77afe0e92bbe04e7d5 2024-11-19T16:15:40,164 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0d6278f2d6904a77afe0e92bbe04e7d5, entries=150, sequenceid=528, filesize=12.0 K 2024-11-19T16:15:40,165 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for 1601148ee1f598abf2fc9cfff51f933e in 623ms, sequenceid=528, compaction requested=false 2024-11-19T16:15:40,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:40,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:40,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-19T16:15:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-19T16:15:40,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-19T16:15:40,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2370 sec 2024-11-19T16:15:40,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.2460 sec 2024-11-19T16:15:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:40,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:15:40,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:40,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:40,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:40,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:40,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:40,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:40,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/f27538c86cdc4d6aaba531b3f52b6b59 is 50, key is test_row_0/A:col10/1732032940452/Put/seqid=0 2024-11-19T16:15:40,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742167_1343 (size=14741) 2024-11-19T16:15:40,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732033000495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:40,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:40,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032940482, max timestamp=1732032940500Num uniques 859; 2024-11-19T16:15:40,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032940482, max timestamp=1732032940500Num uniques 859; 2024-11-19T16:15:40,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:40,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032940482, max timestamp=1732032940500Num uniques 859; 2024-11-19T16:15:40,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:40,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:40,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732033000537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:40,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732033000539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:40,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732033000546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:40,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:40,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732033000607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:40,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732033000656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:40,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732033000657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:40,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732033000657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:40,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732033000817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:40,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732033000866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:40,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732033000866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732033000867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:40,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/f27538c86cdc4d6aaba531b3f52b6b59 2024-11-19T16:15:40,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/0529b85bc4b646689eef2cbb345f3f36 is 50, key is test_row_0/B:col10/1732032940452/Put/seqid=0 2024-11-19T16:15:40,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742168_1344 (size=12301) 2024-11-19T16:15:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T16:15:41,076 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:41,077 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-19T16:15:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T16:15:41,078 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:41,079 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:41,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:41,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732033001127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:41,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732033001176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:41,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732033001176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732033001176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:41,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T16:15:41,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T16:15:41,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:41,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:41,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:41,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:41,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:41,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/0529b85bc4b646689eef2cbb345f3f36 2024-11-19T16:15:41,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/bbf4ae59674844fca6cce3aecb910a78 is 50, key is test_row_0/C:col10/1732032940452/Put/seqid=0 2024-11-19T16:15:41,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742169_1345 (size=12301) 2024-11-19T16:15:41,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T16:15:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:41,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T16:15:41,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T16:15:41,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:41,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:41,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732033001637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:41,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42540 deadline: 1732033001686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:41,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42542 deadline: 1732033001686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42608 deadline: 1732033001687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:15:41,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T16:15:41,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:41,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:41,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:41,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:41,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T16:15:41,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/bbf4ae59674844fca6cce3aecb910a78 2024-11-19T16:15:41,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/f27538c86cdc4d6aaba531b3f52b6b59 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/f27538c86cdc4d6aaba531b3f52b6b59 2024-11-19T16:15:41,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/f27538c86cdc4d6aaba531b3f52b6b59, entries=200, sequenceid=542, filesize=14.4 K 2024-11-19T16:15:41,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/0529b85bc4b646689eef2cbb345f3f36 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0529b85bc4b646689eef2cbb345f3f36 2024-11-19T16:15:41,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0529b85bc4b646689eef2cbb345f3f36, entries=150, sequenceid=542, filesize=12.0 K 2024-11-19T16:15:41,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/bbf4ae59674844fca6cce3aecb910a78 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/bbf4ae59674844fca6cce3aecb910a78 2024-11-19T16:15:41,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/bbf4ae59674844fca6cce3aecb910a78, entries=150, sequenceid=542, filesize=12.0 K 2024-11-19T16:15:41,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1601148ee1f598abf2fc9cfff51f933e in 1271ms, sequenceid=542, compaction requested=true 2024-11-19T16:15:41,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:41,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:41,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:41,725 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:41,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:41,725 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:41,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:41,725 INFO [Thread-1144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1144. 2024-11-19T16:15:41,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1601148ee1f598abf2fc9cfff51f933e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:41,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:41,725 DEBUG [Thread-1144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:41,725 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:41,725 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:41,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:41,726 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:41,726 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/A is initiating minor compaction (all files) 2024-11-19T16:15:41,726 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/A in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:41,726 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/aeff524ad5cc4b23afe9b48147c79fb8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/0e96acbb92fc48b5a4f71d1729a2be53, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/f27538c86cdc4d6aaba531b3f52b6b59] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=39.5 K 2024-11-19T16:15:41,726 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:41,726 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/B is initiating minor compaction (all files) 2024-11-19T16:15:41,726 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/B in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:41,726 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/f1930c0a465c47579dcd355e4edca8f5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/7565d077ddc44fb9baf1b3050f29c057, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0529b85bc4b646689eef2cbb345f3f36] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=37.1 K 2024-11-19T16:15:41,726 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeff524ad5cc4b23afe9b48147c79fb8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1732032936088 2024-11-19T16:15:41,726 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e96acbb92fc48b5a4f71d1729a2be53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1732032938287 2024-11-19T16:15:41,726 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting f1930c0a465c47579dcd355e4edca8f5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1732032936088 2024-11-19T16:15:41,727 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting f27538c86cdc4d6aaba531b3f52b6b59, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1732032940448 2024-11-19T16:15:41,727 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 7565d077ddc44fb9baf1b3050f29c057, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1732032938287 2024-11-19T16:15:41,727 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 0529b85bc4b646689eef2cbb345f3f36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1732032940449 2024-11-19T16:15:41,728 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-11-19T16:15:41,728 INFO [Thread-1140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1140. 2024-11-19T16:15:41,728 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:41,728 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:41,728 DEBUG [Thread-1140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:41,728 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:41,728 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:41,728 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:41,728 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:41,728 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:41,732 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-11-19T16:15:41,732 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:41,732 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:41,732 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:41,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:41,733 INFO [Thread-1142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1142. 2024-11-19T16:15:41,733 DEBUG [Thread-1142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:41,734 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:41,734 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:41,734 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:41,736 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#B#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:41,736 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#A#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:41,736 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/54f937e78e214cb1bd773d5d448e89c3 is 50, key is test_row_0/A:col10/1732032940452/Put/seqid=0 2024-11-19T16:15:41,736 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/4255bf28c3004c8182c61e5347f91f42 is 50, key is test_row_0/B:col10/1732032940452/Put/seqid=0 2024-11-19T16:15:41,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742170_1346 (size=13459) 2024-11-19T16:15:41,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742171_1347 (size=13459) 2024-11-19T16:15:41,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T16:15:41,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:41,845 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T16:15:41,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:41,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:41,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:41,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:41,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:41,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:41,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/b974f99a41484820a7cac19a623f0ca6 is 50, key is test_row_0/A:col10/1732032940536/Put/seqid=0 2024-11-19T16:15:41,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742172_1348 (size=12301) 2024-11-19T16:15:42,149 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/54f937e78e214cb1bd773d5d448e89c3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/54f937e78e214cb1bd773d5d448e89c3 2024-11-19T16:15:42,149 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/4255bf28c3004c8182c61e5347f91f42 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4255bf28c3004c8182c61e5347f91f42 2024-11-19T16:15:42,154 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/A of 1601148ee1f598abf2fc9cfff51f933e into 54f937e78e214cb1bd773d5d448e89c3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:42,154 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/B of 1601148ee1f598abf2fc9cfff51f933e into 4255bf28c3004c8182c61e5347f91f42(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:42,154 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:42,154 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:42,154 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/A, priority=13, startTime=1732032941725; duration=0sec 2024-11-19T16:15:42,154 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/B, priority=13, startTime=1732032941725; duration=0sec 2024-11-19T16:15:42,154 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:42,154 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:A 2024-11-19T16:15:42,154 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:42,155 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:B 2024-11-19T16:15:42,155 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:42,155 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:42,155 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 1601148ee1f598abf2fc9cfff51f933e/C is initiating minor compaction (all files) 2024-11-19T16:15:42,155 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1601148ee1f598abf2fc9cfff51f933e/C in TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:42,156 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/da85cc23e45c4a3dba01b63223f66adf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0d6278f2d6904a77afe0e92bbe04e7d5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/bbf4ae59674844fca6cce3aecb910a78] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp, totalSize=37.1 K 2024-11-19T16:15:42,156 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting da85cc23e45c4a3dba01b63223f66adf, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1732032936088 2024-11-19T16:15:42,156 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d6278f2d6904a77afe0e92bbe04e7d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1732032938287 2024-11-19T16:15:42,156 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbf4ae59674844fca6cce3aecb910a78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1732032940449 2024-11-19T16:15:42,162 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1601148ee1f598abf2fc9cfff51f933e#C#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:42,162 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/c39ebe4b3707415cbe5ef77ebe735f6c is 50, key is test_row_0/C:col10/1732032940452/Put/seqid=0 2024-11-19T16:15:42,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742173_1349 (size=13459) 2024-11-19T16:15:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T16:15:42,258 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/b974f99a41484820a7cac19a623f0ca6 2024-11-19T16:15:42,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/91271608c6e44a709db9227c91bfd51e is 50, key is test_row_0/B:col10/1732032940536/Put/seqid=0 2024-11-19T16:15:42,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742174_1350 (size=12301) 2024-11-19T16:15:42,300 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T16:15:42,570 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/c39ebe4b3707415cbe5ef77ebe735f6c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/c39ebe4b3707415cbe5ef77ebe735f6c 2024-11-19T16:15:42,574 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1601148ee1f598abf2fc9cfff51f933e/C of 1601148ee1f598abf2fc9cfff51f933e into c39ebe4b3707415cbe5ef77ebe735f6c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:42,574 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:42,574 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., storeName=1601148ee1f598abf2fc9cfff51f933e/C, priority=13, startTime=1732032941725; duration=0sec 2024-11-19T16:15:42,574 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:42,574 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1601148ee1f598abf2fc9cfff51f933e:C 2024-11-19T16:15:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:42,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. as already flushing 2024-11-19T16:15:42,658 INFO [Thread-1135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1135. 2024-11-19T16:15:42,658 DEBUG [Thread-1135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:42,658 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:42,658 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:42,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:42,673 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/91271608c6e44a709db9227c91bfd51e 2024-11-19T16:15:42,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/b23daf11242a4135a281e3ca3d7b34a9 is 50, key is test_row_0/C:col10/1732032940536/Put/seqid=0 2024-11-19T16:15:42,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742175_1351 (size=12301) 2024-11-19T16:15:42,699 INFO [Thread-1133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1133. 2024-11-19T16:15:42,699 DEBUG [Thread-1133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:42,699 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:42,699 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:42,699 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:42,709 INFO [Thread-1131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1131. 2024-11-19T16:15:42,709 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-19T16:15:42,709 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:42,709 DEBUG [Thread-1131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:42,709 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:42,709 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:42,709 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:42,709 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:42,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:42,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:43,085 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/b23daf11242a4135a281e3ca3d7b34a9 2024-11-19T16:15:43,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/b974f99a41484820a7cac19a623f0ca6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/b974f99a41484820a7cac19a623f0ca6 2024-11-19T16:15:43,103 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/b974f99a41484820a7cac19a623f0ca6, entries=150, sequenceid=577, filesize=12.0 K 2024-11-19T16:15:43,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/91271608c6e44a709db9227c91bfd51e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/91271608c6e44a709db9227c91bfd51e 2024-11-19T16:15:43,110 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/91271608c6e44a709db9227c91bfd51e, entries=150, sequenceid=577, filesize=12.0 K 2024-11-19T16:15:43,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/b23daf11242a4135a281e3ca3d7b34a9 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b23daf11242a4135a281e3ca3d7b34a9 2024-11-19T16:15:43,114 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b23daf11242a4135a281e3ca3d7b34a9, entries=150, sequenceid=577, filesize=12.0 K 2024-11-19T16:15:43,114 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=26.84 KB/27480 for 1601148ee1f598abf2fc9cfff51f933e in 1269ms, sequenceid=577, compaction requested=false 2024-11-19T16:15:43,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 1601148ee1f598abf2fc9cfff51f933e: 2024-11-19T16:15:43,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:43,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-19T16:15:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-19T16:15:43,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-19T16:15:43,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0370 sec 2024-11-19T16:15:43,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.0400 sec 2024-11-19T16:15:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T16:15:43,237 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:46,779 INFO [Thread-1129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1129. 2024-11-19T16:15:46,779 DEBUG [Thread-1129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:15:46,779 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:46,779 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:46,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:46,779 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T16:15:46,779 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-19T16:15:46,779 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-19T16:15:46,779 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-19T16:15:46,779 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-19T16:15:46,779 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-19T16:15:46,779 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T16:15:46,779 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2901 2024-11-19T16:15:46,779 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2853 2024-11-19T16:15:46,779 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2795 2024-11-19T16:15:46,779 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2877 2024-11-19T16:15:46,779 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2832 2024-11-19T16:15:46,779 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T16:15:46,779 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T16:15:46,780 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T16:15:46,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:46,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:46,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:15:46,780 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-19T16:15:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T16:15:46,783 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032946783"}]},"ts":"1732032946783"} 2024-11-19T16:15:46,784 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T16:15:46,784 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T16:15:46,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T16:15:46,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1601148ee1f598abf2fc9cfff51f933e, UNASSIGN}] 2024-11-19T16:15:46,787 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1601148ee1f598abf2fc9cfff51f933e, UNASSIGN 2024-11-19T16:15:46,787 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=1601148ee1f598abf2fc9cfff51f933e, regionState=CLOSING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:46,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1601148ee1f598abf2fc9cfff51f933e, UNASSIGN because future has completed 2024-11-19T16:15:46,788 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T16:15:46,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:15:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T16:15:46,941 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:46,941 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T16:15:46,941 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 1601148ee1f598abf2fc9cfff51f933e, disabling compactions & flushes 2024-11-19T16:15:46,941 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:46,941 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:46,941 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. after waiting 0 ms 2024-11-19T16:15:46,941 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:46,941 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 1601148ee1f598abf2fc9cfff51f933e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T16:15:46,941 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=A 2024-11-19T16:15:46,941 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:46,941 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=B 2024-11-19T16:15:46,941 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:46,941 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1601148ee1f598abf2fc9cfff51f933e, store=C 2024-11-19T16:15:46,941 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:46,944 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/4b61dc11df6b421faa800b5041e392ff is 50, key is test_row_0/A:col10/1732032942657/Put/seqid=0 2024-11-19T16:15:46,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742176_1352 (size=12301) 2024-11-19T16:15:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T16:15:47,348 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/4b61dc11df6b421faa800b5041e392ff 2024-11-19T16:15:47,360 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/63850aee4c9b4b79856dc880654f7d07 is 50, key is test_row_0/B:col10/1732032942657/Put/seqid=0 2024-11-19T16:15:47,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742177_1353 (size=12301) 2024-11-19T16:15:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T16:15:47,766 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/63850aee4c9b4b79856dc880654f7d07 2024-11-19T16:15:47,778 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/2eb1ff4c94fc44aaaa6e72777c60a9a8 is 50, key is test_row_0/C:col10/1732032942657/Put/seqid=0 2024-11-19T16:15:47,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742178_1354 (size=12301) 2024-11-19T16:15:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T16:15:48,184 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/2eb1ff4c94fc44aaaa6e72777c60a9a8 2024-11-19T16:15:48,190 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/A/4b61dc11df6b421faa800b5041e392ff as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/4b61dc11df6b421faa800b5041e392ff 2024-11-19T16:15:48,195 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/4b61dc11df6b421faa800b5041e392ff, entries=150, sequenceid=588, filesize=12.0 K 2024-11-19T16:15:48,196 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/B/63850aee4c9b4b79856dc880654f7d07 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/63850aee4c9b4b79856dc880654f7d07 2024-11-19T16:15:48,201 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/63850aee4c9b4b79856dc880654f7d07, entries=150, sequenceid=588, filesize=12.0 K 2024-11-19T16:15:48,202 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/.tmp/C/2eb1ff4c94fc44aaaa6e72777c60a9a8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2eb1ff4c94fc44aaaa6e72777c60a9a8 2024-11-19T16:15:48,207 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2eb1ff4c94fc44aaaa6e72777c60a9a8, entries=150, sequenceid=588, filesize=12.0 K 2024-11-19T16:15:48,208 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1601148ee1f598abf2fc9cfff51f933e in 1267ms, sequenceid=588, compaction requested=true 2024-11-19T16:15:48,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/2885864f6fd14d26bbbb43b3754aee52, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/24d991694b03481eb8a117a905eb5b9f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/870a35566da047d29921cf9681d3dafe, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/509e5cac03434c95bd3bab39849ea1e1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/822c166c429f44f1b23404f0351de81a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/381324ed460a4a818fc5122f99fde1bf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9f3789adff6f4b28a09a3d59da2c85a6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/a99b48c8a8354efb806b9bd93f37c6b9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/09f25293ba824742b39a466494f1aa7f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/30cfe8187f094756bb00c2f806eb343b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/117a37d48bf44e9289ce9254929dd797, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/8fac6a1dafd4460ea566439ed1a154bf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/973180b9b7b848b09fa151bbdf578642, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9ed68942b69b49ed938999f6a37b732c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/ab634cd18a3248279ee5194eb9afaaac, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3228ee679a2546398ebe6f07bd984b48, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/de6fc450fe44405abe4832b6b6223f85, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3d0eea2573b249eeba4428dc145ad52a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/2976f248c4b74c92bdd76f4cc329abd8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3497ead5010b48f6b6bd7a027f2f0572, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9b7ec585a58e4f61bb05718adeb34869, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/d19a1f882d344dec81ab831baac2860b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/6674a5b5c97f452baae49c090a70a34a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/01f0975e995b4162ad7514de5a76864b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/0fceae7eee704e54855bf64329a667d9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/66d8e7317ac940af8a486521c0e9b0e1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/cf7659c035a04097ae9625017242a448, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/33551b38d589444fb9c04ed4a5ccebb1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/d6b13c69c84b4dcf883e8d1eeaca07ea, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3908fdfbb0284c13af38ef8a5553c639, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/aeff524ad5cc4b23afe9b48147c79fb8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/1f35b309e4fd408486cc9091dd600451, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/0e96acbb92fc48b5a4f71d1729a2be53, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/f27538c86cdc4d6aaba531b3f52b6b59] to archive 2024-11-19T16:15:48,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:15:48,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/2885864f6fd14d26bbbb43b3754aee52 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/2885864f6fd14d26bbbb43b3754aee52 2024-11-19T16:15:48,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/24d991694b03481eb8a117a905eb5b9f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/24d991694b03481eb8a117a905eb5b9f 2024-11-19T16:15:48,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/870a35566da047d29921cf9681d3dafe to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/870a35566da047d29921cf9681d3dafe 2024-11-19T16:15:48,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/509e5cac03434c95bd3bab39849ea1e1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/509e5cac03434c95bd3bab39849ea1e1 2024-11-19T16:15:48,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/822c166c429f44f1b23404f0351de81a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/822c166c429f44f1b23404f0351de81a 2024-11-19T16:15:48,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/381324ed460a4a818fc5122f99fde1bf to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/381324ed460a4a818fc5122f99fde1bf 2024-11-19T16:15:48,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9f3789adff6f4b28a09a3d59da2c85a6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9f3789adff6f4b28a09a3d59da2c85a6 2024-11-19T16:15:48,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/a99b48c8a8354efb806b9bd93f37c6b9 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/a99b48c8a8354efb806b9bd93f37c6b9 2024-11-19T16:15:48,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/09f25293ba824742b39a466494f1aa7f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/09f25293ba824742b39a466494f1aa7f 2024-11-19T16:15:48,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/30cfe8187f094756bb00c2f806eb343b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/30cfe8187f094756bb00c2f806eb343b 2024-11-19T16:15:48,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/117a37d48bf44e9289ce9254929dd797 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/117a37d48bf44e9289ce9254929dd797 2024-11-19T16:15:48,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/8fac6a1dafd4460ea566439ed1a154bf to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/8fac6a1dafd4460ea566439ed1a154bf 2024-11-19T16:15:48,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/973180b9b7b848b09fa151bbdf578642 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/973180b9b7b848b09fa151bbdf578642 2024-11-19T16:15:48,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9ed68942b69b49ed938999f6a37b732c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9ed68942b69b49ed938999f6a37b732c 2024-11-19T16:15:48,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/ab634cd18a3248279ee5194eb9afaaac to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/ab634cd18a3248279ee5194eb9afaaac 2024-11-19T16:15:48,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3228ee679a2546398ebe6f07bd984b48 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3228ee679a2546398ebe6f07bd984b48 2024-11-19T16:15:48,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/de6fc450fe44405abe4832b6b6223f85 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/de6fc450fe44405abe4832b6b6223f85 2024-11-19T16:15:48,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3d0eea2573b249eeba4428dc145ad52a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3d0eea2573b249eeba4428dc145ad52a 2024-11-19T16:15:48,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/2976f248c4b74c92bdd76f4cc329abd8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/2976f248c4b74c92bdd76f4cc329abd8 2024-11-19T16:15:48,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3497ead5010b48f6b6bd7a027f2f0572 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3497ead5010b48f6b6bd7a027f2f0572 2024-11-19T16:15:48,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9b7ec585a58e4f61bb05718adeb34869 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/9b7ec585a58e4f61bb05718adeb34869 2024-11-19T16:15:48,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/d19a1f882d344dec81ab831baac2860b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/d19a1f882d344dec81ab831baac2860b 2024-11-19T16:15:48,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/6674a5b5c97f452baae49c090a70a34a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/6674a5b5c97f452baae49c090a70a34a 2024-11-19T16:15:48,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/01f0975e995b4162ad7514de5a76864b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/01f0975e995b4162ad7514de5a76864b 2024-11-19T16:15:48,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/0fceae7eee704e54855bf64329a667d9 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/0fceae7eee704e54855bf64329a667d9 2024-11-19T16:15:48,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/66d8e7317ac940af8a486521c0e9b0e1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/66d8e7317ac940af8a486521c0e9b0e1 2024-11-19T16:15:48,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/cf7659c035a04097ae9625017242a448 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/cf7659c035a04097ae9625017242a448 2024-11-19T16:15:48,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/33551b38d589444fb9c04ed4a5ccebb1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/33551b38d589444fb9c04ed4a5ccebb1 2024-11-19T16:15:48,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/d6b13c69c84b4dcf883e8d1eeaca07ea to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/d6b13c69c84b4dcf883e8d1eeaca07ea 2024-11-19T16:15:48,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3908fdfbb0284c13af38ef8a5553c639 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/3908fdfbb0284c13af38ef8a5553c639 2024-11-19T16:15:48,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/aeff524ad5cc4b23afe9b48147c79fb8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/aeff524ad5cc4b23afe9b48147c79fb8 2024-11-19T16:15:48,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/1f35b309e4fd408486cc9091dd600451 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/1f35b309e4fd408486cc9091dd600451 2024-11-19T16:15:48,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/0e96acbb92fc48b5a4f71d1729a2be53 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/0e96acbb92fc48b5a4f71d1729a2be53 2024-11-19T16:15:48,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/f27538c86cdc4d6aaba531b3f52b6b59 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/f27538c86cdc4d6aaba531b3f52b6b59 2024-11-19T16:15:48,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/b8bf7fb557f745538865bc94ee06ada7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/86fe69b828524701a2452b2e79ed0e3c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/29dde8f15794404398996ffe9a9ce764, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/531b04dcfaa5407ba20612277f134a89, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/c75c1d4bfab74dc1a747f4f3c6c514e2, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0d73916bb0b84089b333344aa0b3862e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/177265ec7fb642f79bd8662505bd65d7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/69821af53a534e16b12e38c72dffceca, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/f81364e113744b45af0fa11f806333b6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/5187643330584e9d969035483ce3c3b3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0764508a07864a5a8f5a21714a49bfe0, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/e5a106caa54d492580e538e9350e7242, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4e09a85feec1420593f1f27660d9996f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/39afad95444f44c7aa8e7bfe98cd1340, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0d363a4782eb4e079411fff0def44527, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4d8948273f264da6809d2f076cabc801, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/220d4fc1af084a1dbcbe713f0e75041c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/b03e8aebc8084c76b0db4ec6781f2fba, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/d37c383cbc824c76a9b3ab3a3c2408f1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/02c1880f914c4849bd8914cc1292bc4b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/7191d36e761049c0b9e89be34b32cc73, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/72a8bb2be2934070ab9fe7fc8bbbba23, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/39df1a9aea904514b2feb037383b74bc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/15d02d2c756d4a50a5db5dc55dfa92e7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/99b16b61068248d4a6bd73710430d7fa, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/fdc01d16a93d467d9b39ca092007a497, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/a8b5722639654c78ad94d5f7ac72e76b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0161b046cfbc4e148f7769e1744d6bd4, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/fc3aa9c02f9a434b9d4a51e0cb25c758, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/29e390e35479441f8893c4506f84c820, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/f1930c0a465c47579dcd355e4edca8f5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/640d85f2957e4a27ad3fc50ed9f8fb9d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/7565d077ddc44fb9baf1b3050f29c057, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0529b85bc4b646689eef2cbb345f3f36] to archive 2024-11-19T16:15:48,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:15:48,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/b8bf7fb557f745538865bc94ee06ada7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/b8bf7fb557f745538865bc94ee06ada7 2024-11-19T16:15:48,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/86fe69b828524701a2452b2e79ed0e3c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/86fe69b828524701a2452b2e79ed0e3c 2024-11-19T16:15:48,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/29dde8f15794404398996ffe9a9ce764 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/29dde8f15794404398996ffe9a9ce764 2024-11-19T16:15:48,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/531b04dcfaa5407ba20612277f134a89 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/531b04dcfaa5407ba20612277f134a89 2024-11-19T16:15:48,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/c75c1d4bfab74dc1a747f4f3c6c514e2 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/c75c1d4bfab74dc1a747f4f3c6c514e2 2024-11-19T16:15:48,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0d73916bb0b84089b333344aa0b3862e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0d73916bb0b84089b333344aa0b3862e 2024-11-19T16:15:48,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/177265ec7fb642f79bd8662505bd65d7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/177265ec7fb642f79bd8662505bd65d7 2024-11-19T16:15:48,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/69821af53a534e16b12e38c72dffceca to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/69821af53a534e16b12e38c72dffceca 2024-11-19T16:15:48,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/f81364e113744b45af0fa11f806333b6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/f81364e113744b45af0fa11f806333b6 2024-11-19T16:15:48,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/5187643330584e9d969035483ce3c3b3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/5187643330584e9d969035483ce3c3b3 2024-11-19T16:15:48,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0764508a07864a5a8f5a21714a49bfe0 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0764508a07864a5a8f5a21714a49bfe0 2024-11-19T16:15:48,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/e5a106caa54d492580e538e9350e7242 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/e5a106caa54d492580e538e9350e7242 2024-11-19T16:15:48,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4e09a85feec1420593f1f27660d9996f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4e09a85feec1420593f1f27660d9996f 2024-11-19T16:15:48,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/39afad95444f44c7aa8e7bfe98cd1340 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/39afad95444f44c7aa8e7bfe98cd1340 2024-11-19T16:15:48,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0d363a4782eb4e079411fff0def44527 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0d363a4782eb4e079411fff0def44527 2024-11-19T16:15:48,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4d8948273f264da6809d2f076cabc801 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4d8948273f264da6809d2f076cabc801 2024-11-19T16:15:48,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/220d4fc1af084a1dbcbe713f0e75041c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/220d4fc1af084a1dbcbe713f0e75041c 2024-11-19T16:15:48,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/b03e8aebc8084c76b0db4ec6781f2fba to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/b03e8aebc8084c76b0db4ec6781f2fba 2024-11-19T16:15:48,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/d37c383cbc824c76a9b3ab3a3c2408f1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/d37c383cbc824c76a9b3ab3a3c2408f1 2024-11-19T16:15:48,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/02c1880f914c4849bd8914cc1292bc4b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/02c1880f914c4849bd8914cc1292bc4b 2024-11-19T16:15:48,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/7191d36e761049c0b9e89be34b32cc73 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/7191d36e761049c0b9e89be34b32cc73 2024-11-19T16:15:48,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/72a8bb2be2934070ab9fe7fc8bbbba23 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/72a8bb2be2934070ab9fe7fc8bbbba23 2024-11-19T16:15:48,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/39df1a9aea904514b2feb037383b74bc to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/39df1a9aea904514b2feb037383b74bc 2024-11-19T16:15:48,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/15d02d2c756d4a50a5db5dc55dfa92e7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/15d02d2c756d4a50a5db5dc55dfa92e7 2024-11-19T16:15:48,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/99b16b61068248d4a6bd73710430d7fa to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/99b16b61068248d4a6bd73710430d7fa 2024-11-19T16:15:48,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/fdc01d16a93d467d9b39ca092007a497 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/fdc01d16a93d467d9b39ca092007a497 2024-11-19T16:15:48,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/a8b5722639654c78ad94d5f7ac72e76b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/a8b5722639654c78ad94d5f7ac72e76b 2024-11-19T16:15:48,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0161b046cfbc4e148f7769e1744d6bd4 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0161b046cfbc4e148f7769e1744d6bd4 2024-11-19T16:15:48,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/fc3aa9c02f9a434b9d4a51e0cb25c758 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/fc3aa9c02f9a434b9d4a51e0cb25c758 2024-11-19T16:15:48,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/29e390e35479441f8893c4506f84c820 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/29e390e35479441f8893c4506f84c820 2024-11-19T16:15:48,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/f1930c0a465c47579dcd355e4edca8f5 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/f1930c0a465c47579dcd355e4edca8f5 2024-11-19T16:15:48,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/640d85f2957e4a27ad3fc50ed9f8fb9d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/640d85f2957e4a27ad3fc50ed9f8fb9d 2024-11-19T16:15:48,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/7565d077ddc44fb9baf1b3050f29c057 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/7565d077ddc44fb9baf1b3050f29c057 2024-11-19T16:15:48,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0529b85bc4b646689eef2cbb345f3f36 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/0529b85bc4b646689eef2cbb345f3f36 2024-11-19T16:15:48,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/72204856534d4f059fbc8753db93b207, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2de55cd250ce461c9887122349326132, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/21fd218e3e0f4be89cef6bba4bbed1b7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/911f44fedc7745d8a806850ece8500d8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/8436541e846a4eb8820da52ca630112a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/20b45d8e5c4c49a891a338e2300a244f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/89451ae47ea74bc8b5a8cefd59c63644, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/3c21b8fd8cc64d93847a1be8debb5f2e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/e6086d9b81264e689bfc758a16930c5d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/7e636a9cb5d9415ba5b0c2c136a2ac29, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/acbf4b171c5f47048a347830c8530f8c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/3bd319dbb5354af9b41fe759bb4f2506, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/4ffe8440e68948bfb0ef363655f6ed06, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/7310b89a0cdf409d924dfe1858a85854, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2de20497b3614c6b9ba46c9ddb452009, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/953773a9d17b4faea4bc69ebfbf51579, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/169b13460c44449e87f5a65f1af46249, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a45348b23bfd459286da3bcfece6a3b3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a3b6f9ca00ea4e509f6ed00dc814693b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/cc66d1c5a4e94c9096f71314836a6891, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0e3185bd07ab4ff79783fa357b5e2118, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/82b3114aebb0436784cb6e10d5f72d6c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b142701af38c4a3ca245f6017f46e772, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b7951d978ef34e8c9d2f6220341936aa, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/ab0b9c5ad7284362becfa6c3bc49d217, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/bc7a07c16dd44035bc025d9892ed9604, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a33155433c39450182a2b70915f4771c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/e800baa292e5423283133d69b3f37ad4, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/51d801acf1ba415e9fa5b8fc04cd55c4, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/41a0bc9924db4fdc819dcc673061ff4d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/da85cc23e45c4a3dba01b63223f66adf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0d0db7f382a24c19a32b7115e8a3db1f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0d6278f2d6904a77afe0e92bbe04e7d5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/bbf4ae59674844fca6cce3aecb910a78] to archive 2024-11-19T16:15:48,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:15:48,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/72204856534d4f059fbc8753db93b207 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/72204856534d4f059fbc8753db93b207 2024-11-19T16:15:48,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2de55cd250ce461c9887122349326132 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2de55cd250ce461c9887122349326132 2024-11-19T16:15:48,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/21fd218e3e0f4be89cef6bba4bbed1b7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/21fd218e3e0f4be89cef6bba4bbed1b7 2024-11-19T16:15:48,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/911f44fedc7745d8a806850ece8500d8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/911f44fedc7745d8a806850ece8500d8 2024-11-19T16:15:48,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/8436541e846a4eb8820da52ca630112a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/8436541e846a4eb8820da52ca630112a 2024-11-19T16:15:48,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/20b45d8e5c4c49a891a338e2300a244f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/20b45d8e5c4c49a891a338e2300a244f 2024-11-19T16:15:48,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/89451ae47ea74bc8b5a8cefd59c63644 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/89451ae47ea74bc8b5a8cefd59c63644 2024-11-19T16:15:48,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/3c21b8fd8cc64d93847a1be8debb5f2e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/3c21b8fd8cc64d93847a1be8debb5f2e 2024-11-19T16:15:48,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/e6086d9b81264e689bfc758a16930c5d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/e6086d9b81264e689bfc758a16930c5d 2024-11-19T16:15:48,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/7e636a9cb5d9415ba5b0c2c136a2ac29 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/7e636a9cb5d9415ba5b0c2c136a2ac29 2024-11-19T16:15:48,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/acbf4b171c5f47048a347830c8530f8c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/acbf4b171c5f47048a347830c8530f8c 2024-11-19T16:15:48,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/3bd319dbb5354af9b41fe759bb4f2506 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/3bd319dbb5354af9b41fe759bb4f2506 2024-11-19T16:15:48,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/4ffe8440e68948bfb0ef363655f6ed06 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/4ffe8440e68948bfb0ef363655f6ed06 2024-11-19T16:15:48,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/7310b89a0cdf409d924dfe1858a85854 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/7310b89a0cdf409d924dfe1858a85854 2024-11-19T16:15:48,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2de20497b3614c6b9ba46c9ddb452009 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2de20497b3614c6b9ba46c9ddb452009 2024-11-19T16:15:48,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/953773a9d17b4faea4bc69ebfbf51579 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/953773a9d17b4faea4bc69ebfbf51579 2024-11-19T16:15:48,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/169b13460c44449e87f5a65f1af46249 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/169b13460c44449e87f5a65f1af46249 2024-11-19T16:15:48,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a45348b23bfd459286da3bcfece6a3b3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a45348b23bfd459286da3bcfece6a3b3 2024-11-19T16:15:48,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a3b6f9ca00ea4e509f6ed00dc814693b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a3b6f9ca00ea4e509f6ed00dc814693b 2024-11-19T16:15:48,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/cc66d1c5a4e94c9096f71314836a6891 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/cc66d1c5a4e94c9096f71314836a6891 2024-11-19T16:15:48,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0e3185bd07ab4ff79783fa357b5e2118 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0e3185bd07ab4ff79783fa357b5e2118 2024-11-19T16:15:48,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/82b3114aebb0436784cb6e10d5f72d6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/82b3114aebb0436784cb6e10d5f72d6c 2024-11-19T16:15:48,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b142701af38c4a3ca245f6017f46e772 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b142701af38c4a3ca245f6017f46e772 2024-11-19T16:15:48,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b7951d978ef34e8c9d2f6220341936aa to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b7951d978ef34e8c9d2f6220341936aa 2024-11-19T16:15:48,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/ab0b9c5ad7284362becfa6c3bc49d217 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/ab0b9c5ad7284362becfa6c3bc49d217 2024-11-19T16:15:48,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/bc7a07c16dd44035bc025d9892ed9604 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/bc7a07c16dd44035bc025d9892ed9604 2024-11-19T16:15:48,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a33155433c39450182a2b70915f4771c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/a33155433c39450182a2b70915f4771c 2024-11-19T16:15:48,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/e800baa292e5423283133d69b3f37ad4 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/e800baa292e5423283133d69b3f37ad4 2024-11-19T16:15:48,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/51d801acf1ba415e9fa5b8fc04cd55c4 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/51d801acf1ba415e9fa5b8fc04cd55c4 2024-11-19T16:15:48,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/41a0bc9924db4fdc819dcc673061ff4d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/41a0bc9924db4fdc819dcc673061ff4d 2024-11-19T16:15:48,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/da85cc23e45c4a3dba01b63223f66adf to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/da85cc23e45c4a3dba01b63223f66adf 2024-11-19T16:15:48,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0d0db7f382a24c19a32b7115e8a3db1f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0d0db7f382a24c19a32b7115e8a3db1f 2024-11-19T16:15:48,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0d6278f2d6904a77afe0e92bbe04e7d5 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/0d6278f2d6904a77afe0e92bbe04e7d5 2024-11-19T16:15:48,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/bbf4ae59674844fca6cce3aecb910a78 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/bbf4ae59674844fca6cce3aecb910a78 2024-11-19T16:15:48,322 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/recovered.edits/591.seqid, newMaxSeqId=591, maxSeqId=1 2024-11-19T16:15:48,322 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. 2024-11-19T16:15:48,322 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 1601148ee1f598abf2fc9cfff51f933e: Waiting for close lock at 1732032946941Running coprocessor pre-close hooks at 1732032946941Disabling compacts and flushes for region at 1732032946941Disabling writes for close at 1732032946941Obtaining lock to block concurrent updates at 1732032946941Preparing flush snapshotting stores in 1601148ee1f598abf2fc9cfff51f933e at 1732032946941Finished memstore snapshotting TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732032946941Flushing stores of TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e. at 1732032946942 (+1 ms)Flushing 1601148ee1f598abf2fc9cfff51f933e/A: creating writer at 1732032946942Flushing 1601148ee1f598abf2fc9cfff51f933e/A: appending metadata at 1732032946944 (+2 ms)Flushing 1601148ee1f598abf2fc9cfff51f933e/A: closing flushed file at 1732032946944Flushing 1601148ee1f598abf2fc9cfff51f933e/B: creating writer at 1732032947354 (+410 ms)Flushing 1601148ee1f598abf2fc9cfff51f933e/B: appending metadata at 1732032947359 (+5 ms)Flushing 1601148ee1f598abf2fc9cfff51f933e/B: closing flushed file at 1732032947359Flushing 1601148ee1f598abf2fc9cfff51f933e/C: creating writer at 1732032947774 (+415 ms)Flushing 1601148ee1f598abf2fc9cfff51f933e/C: appending metadata at 1732032947778 (+4 ms)Flushing 1601148ee1f598abf2fc9cfff51f933e/C: closing flushed file at 1732032947778Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@256ccd52: reopening flushed file at 1732032948189 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50a73d60: reopening flushed file at 1732032948195 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53c15149: reopening flushed file at 1732032948201 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1601148ee1f598abf2fc9cfff51f933e in 1267ms, sequenceid=588, compaction requested=true at 1732032948208 (+7 ms)Writing region close event to WAL at 1732032948319 (+111 ms)Running coprocessor post-close hooks at 1732032948322 (+3 ms)Closed at 1732032948322 2024-11-19T16:15:48,324 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:48,324 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=1601148ee1f598abf2fc9cfff51f933e, regionState=CLOSED 2024-11-19T16:15:48,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:15:48,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-19T16:15:48,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 1601148ee1f598abf2fc9cfff51f933e, server=32a8860bb9d6,43095,1732032854050 in 1.5370 sec 2024-11-19T16:15:48,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-19T16:15:48,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1601148ee1f598abf2fc9cfff51f933e, UNASSIGN in 1.5420 sec 2024-11-19T16:15:48,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-19T16:15:48,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5440 sec 2024-11-19T16:15:48,332 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032948331"}]},"ts":"1732032948331"} 2024-11-19T16:15:48,333 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T16:15:48,333 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T16:15:48,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5530 sec 2024-11-19T16:15:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T16:15:48,927 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:48,928 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-19T16:15:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:48,929 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T16:15:48,930 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:48,932 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:48,934 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/recovered.edits] 2024-11-19T16:15:48,938 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/4b61dc11df6b421faa800b5041e392ff to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/4b61dc11df6b421faa800b5041e392ff 2024-11-19T16:15:48,941 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/54f937e78e214cb1bd773d5d448e89c3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/54f937e78e214cb1bd773d5d448e89c3 2024-11-19T16:15:48,943 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/b974f99a41484820a7cac19a623f0ca6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/A/b974f99a41484820a7cac19a623f0ca6 2024-11-19T16:15:48,948 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4255bf28c3004c8182c61e5347f91f42 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/4255bf28c3004c8182c61e5347f91f42 2024-11-19T16:15:48,949 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/63850aee4c9b4b79856dc880654f7d07 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/63850aee4c9b4b79856dc880654f7d07 2024-11-19T16:15:48,951 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/91271608c6e44a709db9227c91bfd51e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/B/91271608c6e44a709db9227c91bfd51e 2024-11-19T16:15:48,953 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2eb1ff4c94fc44aaaa6e72777c60a9a8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/2eb1ff4c94fc44aaaa6e72777c60a9a8 2024-11-19T16:15:48,954 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b23daf11242a4135a281e3ca3d7b34a9 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/b23daf11242a4135a281e3ca3d7b34a9 2024-11-19T16:15:48,958 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/c39ebe4b3707415cbe5ef77ebe735f6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/C/c39ebe4b3707415cbe5ef77ebe735f6c 2024-11-19T16:15:48,962 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/recovered.edits/591.seqid to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e/recovered.edits/591.seqid 2024-11-19T16:15:48,964 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/1601148ee1f598abf2fc9cfff51f933e 2024-11-19T16:15:48,964 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T16:15:48,967 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:48,971 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T16:15:48,973 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T16:15:48,975 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:48,975 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T16:15:48,975 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732032948975"}]},"ts":"9223372036854775807"} 2024-11-19T16:15:48,977 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T16:15:48,977 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1601148ee1f598abf2fc9cfff51f933e, NAME => 'TestAcidGuarantees,,1732032919490.1601148ee1f598abf2fc9cfff51f933e.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T16:15:48,977 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T16:15:48,978 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732032948977"}]},"ts":"9223372036854775807"} 2024-11-19T16:15:48,980 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T16:15:48,981 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:48,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-19T16:15:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T16:15:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T16:15:49,042 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:49,051 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=247 (was 248), OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=424 (was 344) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4575 (was 4172) - AvailableMemoryMB LEAK? - 2024-11-19T16:15:49,063 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=424, ProcessCount=11, AvailableMemoryMB=4574 2024-11-19T16:15:49,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T16:15:49,064 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T16:15:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:49,067 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T16:15:49,067 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:49,068 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-19T16:15:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T16:15:49,069 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T16:15:49,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742179_1355 (size=1061) 2024-11-19T16:15:49,085 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70 2024-11-19T16:15:49,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742180_1356 (size=53) 2024-11-19T16:15:49,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T16:15:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T16:15:49,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:15:49,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 47bdefb8e31bb07f134a6eace775a473, disabling compactions & flushes 2024-11-19T16:15:49,496 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:49,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:49,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. after waiting 0 ms 2024-11-19T16:15:49,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:49,496 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:49,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 47bdefb8e31bb07f134a6eace775a473: Waiting for close lock at 1732032949496Disabling compacts and flushes for region at 1732032949496Disabling writes for close at 1732032949496Writing region close event to WAL at 1732032949496Closed at 1732032949496 2024-11-19T16:15:49,529 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T16:15:49,529 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732032949529"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732032949529"}]},"ts":"1732032949529"} 2024-11-19T16:15:49,531 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T16:15:49,532 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T16:15:49,532 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032949532"}]},"ts":"1732032949532"} 2024-11-19T16:15:49,533 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T16:15:49,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, ASSIGN}] 2024-11-19T16:15:49,535 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, ASSIGN 2024-11-19T16:15:49,535 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, ASSIGN; state=OFFLINE, location=32a8860bb9d6,43095,1732032854050; forceNewPlan=false, retain=false 2024-11-19T16:15:49,686 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=47bdefb8e31bb07f134a6eace775a473, regionState=OPENING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:49,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, ASSIGN because future has completed 2024-11-19T16:15:49,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:15:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T16:15:49,850 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:49,850 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} 2024-11-19T16:15:49,851 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:49,851 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:15:49,851 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:49,851 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:49,853 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:49,855 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:15:49,855 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47bdefb8e31bb07f134a6eace775a473 columnFamilyName A 2024-11-19T16:15:49,855 DEBUG [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:49,856 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.HStore(327): Store=47bdefb8e31bb07f134a6eace775a473/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:15:49,856 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:49,857 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:15:49,857 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47bdefb8e31bb07f134a6eace775a473 columnFamilyName B 2024-11-19T16:15:49,857 DEBUG [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:49,858 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.HStore(327): Store=47bdefb8e31bb07f134a6eace775a473/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:15:49,858 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:49,859 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:15:49,859 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47bdefb8e31bb07f134a6eace775a473 columnFamilyName C 2024-11-19T16:15:49,859 DEBUG [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:49,859 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.HStore(327): Store=47bdefb8e31bb07f134a6eace775a473/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:15:49,860 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:49,860 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:49,860 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:49,861 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:49,861 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:49,861 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:49,862 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T16:15:49,863 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:49,864 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T16:15:49,865 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 47bdefb8e31bb07f134a6eace775a473; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73328415, jitterRate=0.09267853200435638}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T16:15:49,865 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:49,865 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 47bdefb8e31bb07f134a6eace775a473: Running coprocessor pre-open hook at 1732032949851Writing region info on filesystem at 1732032949851Initializing all the Stores at 1732032949853 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032949853Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032949853Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032949853Cleaning up temporary data from old regions at 1732032949861 (+8 ms)Running coprocessor post-open hooks at 1732032949865 (+4 ms)Region opened successfully at 1732032949865 2024-11-19T16:15:49,866 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., pid=89, masterSystemTime=1732032949845 2024-11-19T16:15:49,867 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:49,867 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:49,868 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=47bdefb8e31bb07f134a6eace775a473, regionState=OPEN, openSeqNum=2, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:49,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:15:49,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-19T16:15:49,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 in 180 msec 2024-11-19T16:15:49,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-19T16:15:49,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, ASSIGN in 338 msec 2024-11-19T16:15:49,874 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T16:15:49,874 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032949874"}]},"ts":"1732032949874"} 2024-11-19T16:15:49,875 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T16:15:49,876 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T16:15:49,877 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 812 msec 2024-11-19T16:15:50,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T16:15:50,208 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:50,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:50,210 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:50,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:50,212 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0c5f70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:50,215 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5694e49d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:50,216 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:50,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:50,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:50,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:50,219 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T16:15:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 32a8860bb9d6,37355,1732032853237 2024-11-19T16:15:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@12491736 2024-11-19T16:15:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T16:15:50,221 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T16:15:50,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T16:15:50,223 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T16:15:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T16:15:50,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742181_1357 (size=1097) 2024-11-19T16:15:50,637 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T16:15:50,637 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T16:15:50,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T16:15:50,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, REOPEN/MOVE}] 2024-11-19T16:15:50,647 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, REOPEN/MOVE 2024-11-19T16:15:50,648 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=47bdefb8e31bb07f134a6eace775a473, regionState=CLOSING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:50,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, REOPEN/MOVE because future has completed 2024-11-19T16:15:50,650 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T16:15:50,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:15:50,803 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:50,803 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T16:15:50,803 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 47bdefb8e31bb07f134a6eace775a473, disabling compactions & flushes 2024-11-19T16:15:50,804 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:50,804 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:50,804 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. after waiting 0 ms 2024-11-19T16:15:50,804 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:50,812 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T16:15:50,813 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:50,813 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 47bdefb8e31bb07f134a6eace775a473: Waiting for close lock at 1732032950803Running coprocessor pre-close hooks at 1732032950803Disabling compacts and flushes for region at 1732032950803Disabling writes for close at 1732032950804 (+1 ms)Writing region close event to WAL at 1732032950805 (+1 ms)Running coprocessor post-close hooks at 1732032950813 (+8 ms)Closed at 1732032950813 2024-11-19T16:15:50,813 WARN [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 47bdefb8e31bb07f134a6eace775a473 to self. 2024-11-19T16:15:50,815 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:50,816 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=47bdefb8e31bb07f134a6eace775a473, regionState=CLOSED 2024-11-19T16:15:50,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:15:50,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-19T16:15:50,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 in 168 msec 2024-11-19T16:15:50,821 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, REOPEN/MOVE; state=CLOSED, location=32a8860bb9d6,43095,1732032854050; forceNewPlan=false, retain=true 2024-11-19T16:15:50,972 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=47bdefb8e31bb07f134a6eace775a473, regionState=OPENING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:50,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, REOPEN/MOVE because future has completed 2024-11-19T16:15:50,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:15:51,134 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,134 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} 2024-11-19T16:15:51,134 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,134 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:15:51,135 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,135 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,137 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,138 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:15:51,138 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47bdefb8e31bb07f134a6eace775a473 columnFamilyName A 2024-11-19T16:15:51,140 DEBUG [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:51,141 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.HStore(327): Store=47bdefb8e31bb07f134a6eace775a473/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:15:51,142 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,142 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:15:51,143 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47bdefb8e31bb07f134a6eace775a473 columnFamilyName B 2024-11-19T16:15:51,143 DEBUG [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:51,143 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.HStore(327): Store=47bdefb8e31bb07f134a6eace775a473/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:15:51,143 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,144 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:15:51,144 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47bdefb8e31bb07f134a6eace775a473 columnFamilyName C 2024-11-19T16:15:51,144 DEBUG [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:51,145 INFO [StoreOpener-47bdefb8e31bb07f134a6eace775a473-1 {}] regionserver.HStore(327): Store=47bdefb8e31bb07f134a6eace775a473/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:15:51,145 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,145 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,146 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,147 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,148 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,148 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,148 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T16:15:51,149 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,150 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 47bdefb8e31bb07f134a6eace775a473; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59631115, jitterRate=-0.11142714321613312}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T16:15:51,150 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,151 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 47bdefb8e31bb07f134a6eace775a473: Running coprocessor pre-open hook at 1732032951135Writing region info on filesystem at 1732032951135Initializing all the Stores at 1732032951136 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032951136Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032951136Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032951136Cleaning up temporary data from old regions at 1732032951148 (+12 ms)Running coprocessor post-open hooks at 1732032951150 (+2 ms)Region opened successfully at 1732032951151 (+1 ms) 2024-11-19T16:15:51,151 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., pid=94, masterSystemTime=1732032951130 2024-11-19T16:15:51,153 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,153 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,153 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=47bdefb8e31bb07f134a6eace775a473, regionState=OPEN, openSeqNum=5, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:15:51,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-19T16:15:51,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 in 179 msec 2024-11-19T16:15:51,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-19T16:15:51,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, REOPEN/MOVE in 510 msec 2024-11-19T16:15:51,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-19T16:15:51,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 515 msec 2024-11-19T16:15:51,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 936 msec 2024-11-19T16:15:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T16:15:51,162 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:51,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:51,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:51,164 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e3192, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:51,165 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a2772d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:51,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:51,167 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3059b9a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:51,168 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7edabc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:51,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:51,169 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c5808e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:51,170 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:51,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fe48072, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:51,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:51,172 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:51,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:51,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:51,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d35bb67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:51,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:51,173 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@470b8baf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,174 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:51,174 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:51,175 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f17f800, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:51,176 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e800cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:51,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:51,178 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed03523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:51,179 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a697ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,180 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:51,180 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:51,181 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f7d0ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:51,182 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5568b64c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:51,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:51,184 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f122a78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:51,185 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@445a995c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:51,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:51,187 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@359a9ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:51,188 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11bb85a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,190 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:15:51,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:15:51,190 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:15:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:15:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:15:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36ccbf3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:15:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:15:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:15:51,192 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:15:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cfc6366, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:15:51,193 DEBUG [Thread-1635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:51,193 DEBUG [Thread-1629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:51,193 DEBUG [Thread-1620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:51,193 DEBUG [Thread-1633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:51,193 DEBUG [Thread-1637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:51,193 DEBUG [Thread-1631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:51,194 DEBUG [Thread-1618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:51,194 DEBUG [Thread-1622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:51,195 DEBUG [Thread-1624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:51,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:51,195 DEBUG [Thread-1626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:15:51,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:51,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:51,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:51,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:51,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:51,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:51,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:51,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:51,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:15:51,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-19T16:15:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:15:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:15:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:15:51,200 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:15:51,201 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:15:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T16:15:51,202 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:51,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:15:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:15:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:15:51,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:15:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:15:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:15:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:15:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:15:51,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:15:51,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:51,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:15:51,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:51,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:15:51,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:51,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c7d04ddb4226407e91739f281631ccc6_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032951199/Put/seqid=0 2024-11-19T16:15:51,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742182_1358 (size=17034) 2024-11-19T16:15:51,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732032951221, max timestamp=1732032951296Num uniques 859; 2024-11-19T16:15:51,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T16:15:51,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732032951221, max timestamp=1732032951296Num uniques 859; 2024-11-19T16:15:51,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T16:15:51,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732032951221, max timestamp=1732032951296Num uniques 859; 2024-11-19T16:15:51,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T16:15:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T16:15:51,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033011327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033011328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033011330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033011331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033011337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T16:15:51,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:51,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033011446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033011446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033011446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033011446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033011447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T16:15:51,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:51,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T16:15:51,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T16:15:51,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:51,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033011657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033011657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033011657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033011657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033011657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,667 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:51,670 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c7d04ddb4226407e91739f281631ccc6_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c7d04ddb4226407e91739f281631ccc6_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:51,671 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/7fa5c5eb315f4579b793729152af08ad, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:51,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/7fa5c5eb315f4579b793729152af08ad is 175, key is test_row_0/A:col10/1732032951199/Put/seqid=0 2024-11-19T16:15:51,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742183_1359 (size=48139) 2024-11-19T16:15:51,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T16:15:51,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:51,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T16:15:51,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T16:15:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:51,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:51,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033011967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033011967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033011967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033011977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033011977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:52,082 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/7fa5c5eb315f4579b793729152af08ad 2024-11-19T16:15:52,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/12489ca2d1554b23af7dcb934a17cf27 is 50, key is test_row_0/B:col10/1732032951199/Put/seqid=0 2024-11-19T16:15:52,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742184_1360 (size=12001) 2024-11-19T16:15:52,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/12489ca2d1554b23af7dcb934a17cf27 2024-11-19T16:15:52,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T16:15:52,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:52,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:52,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/5b895b4fd2734696b98d567afbe7cefd is 50, key is test_row_0/C:col10/1732032951199/Put/seqid=0 2024-11-19T16:15:52,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742185_1361 (size=12001) 2024-11-19T16:15:52,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T16:15:52,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:52,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:52,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:52,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:52,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T16:15:52,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T16:15:52,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:52,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:52,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:52,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:52,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:52,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033012476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:52,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033012486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:52,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033012486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:52,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033012497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:52,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033012497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:52,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/5b895b4fd2734696b98d567afbe7cefd 2024-11-19T16:15:52,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/7fa5c5eb315f4579b793729152af08ad as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7fa5c5eb315f4579b793729152af08ad 2024-11-19T16:15:52,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7fa5c5eb315f4579b793729152af08ad, entries=250, sequenceid=15, filesize=47.0 K 2024-11-19T16:15:52,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/12489ca2d1554b23af7dcb934a17cf27 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/12489ca2d1554b23af7dcb934a17cf27 2024-11-19T16:15:52,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/12489ca2d1554b23af7dcb934a17cf27, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T16:15:52,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/5b895b4fd2734696b98d567afbe7cefd as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/5b895b4fd2734696b98d567afbe7cefd 2024-11-19T16:15:52,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/5b895b4fd2734696b98d567afbe7cefd, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T16:15:52,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 47bdefb8e31bb07f134a6eace775a473 in 1353ms, sequenceid=15, compaction requested=false 2024-11-19T16:15:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:15:52,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T16:15:52,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:52,582 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-19T16:15:52,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:15:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:15:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:15:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:52,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119865193e9a43a4c56a7c00eef3846f999_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032951335/Put/seqid=0 2024-11-19T16:15:52,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742186_1362 (size=12154) 2024-11-19T16:15:52,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:52,605 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119865193e9a43a4c56a7c00eef3846f999_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119865193e9a43a4c56a7c00eef3846f999_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/54360e624ca547f49b86c74385d013ec, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/54360e624ca547f49b86c74385d013ec is 175, key is test_row_0/A:col10/1732032951335/Put/seqid=0 2024-11-19T16:15:52,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742187_1363 (size=30955) 2024-11-19T16:15:53,012 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/54360e624ca547f49b86c74385d013ec 2024-11-19T16:15:53,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/006fad54409f4f56a649854e910155a6 is 50, key is test_row_0/B:col10/1732032951335/Put/seqid=0 2024-11-19T16:15:53,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742188_1364 (size=12001) 2024-11-19T16:15:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T16:15:53,460 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/006fad54409f4f56a649854e910155a6 2024-11-19T16:15:53,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/7650241d84be4749b1fef22638a49655 is 50, key is test_row_0/C:col10/1732032951335/Put/seqid=0 2024-11-19T16:15:53,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742189_1365 (size=12001) 2024-11-19T16:15:53,488 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/7650241d84be4749b1fef22638a49655 2024-11-19T16:15:53,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:53,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:53,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/54360e624ca547f49b86c74385d013ec as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/54360e624ca547f49b86c74385d013ec 2024-11-19T16:15:53,499 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/54360e624ca547f49b86c74385d013ec, entries=150, sequenceid=48, filesize=30.2 K 2024-11-19T16:15:53,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/006fad54409f4f56a649854e910155a6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/006fad54409f4f56a649854e910155a6 2024-11-19T16:15:53,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,505 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/006fad54409f4f56a649854e910155a6, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T16:15:53,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/7650241d84be4749b1fef22638a49655 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/7650241d84be4749b1fef22638a49655 2024-11-19T16:15:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,509 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/7650241d84be4749b1fef22638a49655, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T16:15:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,510 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=20.13 KB/20610 for 47bdefb8e31bb07f134a6eace775a473 in 928ms, sequenceid=48, compaction requested=false 2024-11-19T16:15:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:15:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-19T16:15:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-19T16:15:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-19T16:15:53,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3100 sec 2024-11-19T16:15:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.3160 sec 2024-11-19T16:15:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:15:53,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:15:53,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:53,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:15:53,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:53,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:15:53,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c6f1e26a33404f0c95fd253f1f04af8c_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032953547/Put/seqid=0 2024-11-19T16:15:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:53,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742190_1366 (size=19474) 2024-11-19T16:15:53,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732032953566, max timestamp=1732032953624Num uniques 859; 2024-11-19T16:15:53,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732032953566, max timestamp=1732032953624Num uniques 859; 2024-11-19T16:15:53,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732032953566, max timestamp=1732032953624Num uniques 859; 2024-11-19T16:15:53,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T16:15:53,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T16:15:53,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T16:15:53,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033013661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:53,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033013663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:53,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033013672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:53,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033013673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:53,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033013675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:53,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033013776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:53,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033013787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:53,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033013787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:53,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033013797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:53,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033013797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:53,853 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T16:15:54,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033013997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,010 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:54,013 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c6f1e26a33404f0c95fd253f1f04af8c_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c6f1e26a33404f0c95fd253f1f04af8c_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:54,013 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/5f9fc138a8a14b529ef2646eacb929fd, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:54,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/5f9fc138a8a14b529ef2646eacb929fd is 175, key is test_row_0/A:col10/1732032953547/Put/seqid=0 2024-11-19T16:15:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742191_1367 (size=56733) 2024-11-19T16:15:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033014007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033014007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033014017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033014017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033014307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033014327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033014327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033014337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033014337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,417 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=58, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/5f9fc138a8a14b529ef2646eacb929fd 2024-11-19T16:15:54,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/308a48d24c8047049f5a070c4d76ac98 is 50, key is test_row_0/B:col10/1732032953547/Put/seqid=0 2024-11-19T16:15:54,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742192_1368 (size=12001) 2024-11-19T16:15:54,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/308a48d24c8047049f5a070c4d76ac98 2024-11-19T16:15:54,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/77c78d0f06a14a74979c8226eb2dfa3d is 50, key is test_row_0/C:col10/1732032953547/Put/seqid=0 2024-11-19T16:15:54,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742193_1369 (size=12001) 2024-11-19T16:15:54,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033014827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/77c78d0f06a14a74979c8226eb2dfa3d 2024-11-19T16:15:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/5f9fc138a8a14b529ef2646eacb929fd as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/5f9fc138a8a14b529ef2646eacb929fd 2024-11-19T16:15:54,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/5f9fc138a8a14b529ef2646eacb929fd, entries=300, sequenceid=58, filesize=55.4 K 2024-11-19T16:15:54,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/308a48d24c8047049f5a070c4d76ac98 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/308a48d24c8047049f5a070c4d76ac98 2024-11-19T16:15:54,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/308a48d24c8047049f5a070c4d76ac98, entries=150, sequenceid=58, filesize=11.7 K 2024-11-19T16:15:54,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/77c78d0f06a14a74979c8226eb2dfa3d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/77c78d0f06a14a74979c8226eb2dfa3d 2024-11-19T16:15:54,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/77c78d0f06a14a74979c8226eb2dfa3d, entries=150, sequenceid=58, filesize=11.7 K 2024-11-19T16:15:54,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 47bdefb8e31bb07f134a6eace775a473 in 1306ms, sequenceid=58, compaction requested=true 2024-11-19T16:15:54,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:15:54,855 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:54,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:54,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:54,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:54,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:54,857 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:54,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:54,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:54,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-19T16:15:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:15:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:15:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:15:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:54,859 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:54,859 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/A is initiating minor compaction (all files) 2024-11-19T16:15:54,859 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/A in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:54,859 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7fa5c5eb315f4579b793729152af08ad, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/54360e624ca547f49b86c74385d013ec, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/5f9fc138a8a14b529ef2646eacb929fd] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=132.6 K 2024-11-19T16:15:54,859 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:54,859 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7fa5c5eb315f4579b793729152af08ad, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/54360e624ca547f49b86c74385d013ec, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/5f9fc138a8a14b529ef2646eacb929fd] 2024-11-19T16:15:54,860 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fa5c5eb315f4579b793729152af08ad, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732032951199 2024-11-19T16:15:54,860 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54360e624ca547f49b86c74385d013ec, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732032951292 2024-11-19T16:15:54,861 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:54,861 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/B is initiating minor compaction (all files) 2024-11-19T16:15:54,861 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f9fc138a8a14b529ef2646eacb929fd, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032953492 2024-11-19T16:15:54,861 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/B in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:54,861 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/12489ca2d1554b23af7dcb934a17cf27, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/006fad54409f4f56a649854e910155a6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/308a48d24c8047049f5a070c4d76ac98] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=35.2 K 2024-11-19T16:15:54,861 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 12489ca2d1554b23af7dcb934a17cf27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732032951199 2024-11-19T16:15:54,862 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 006fad54409f4f56a649854e910155a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732032951292 2024-11-19T16:15:54,863 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 308a48d24c8047049f5a070c4d76ac98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032953492 2024-11-19T16:15:54,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d4cc3820c716491fbbe7ecf65d64ae35_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032953656/Put/seqid=0 2024-11-19T16:15:54,870 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:54,879 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#B#compaction#320 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:54,880 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/f66c3b363e2c4a129c0d1acb814ed4f2 is 50, key is test_row_0/B:col10/1732032953547/Put/seqid=0 2024-11-19T16:15:54,881 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411196aa8eb9dcf7b4a9e9b20411430e19366_47bdefb8e31bb07f134a6eace775a473 store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:54,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033014870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,883 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411196aa8eb9dcf7b4a9e9b20411430e19366_47bdefb8e31bb07f134a6eace775a473, store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:54,883 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196aa8eb9dcf7b4a9e9b20411430e19366_47bdefb8e31bb07f134a6eace775a473 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:54,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033014871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033014875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742194_1370 (size=14594) 2024-11-19T16:15:54,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033014876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:54,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742195_1371 (size=12104) 2024-11-19T16:15:54,908 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/f66c3b363e2c4a129c0d1acb814ed4f2 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/f66c3b363e2c4a129c0d1acb814ed4f2 2024-11-19T16:15:54,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742196_1372 (size=4469) 2024-11-19T16:15:54,915 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/B of 47bdefb8e31bb07f134a6eace775a473 into f66c3b363e2c4a129c0d1acb814ed4f2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:54,915 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:15:54,916 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/B, priority=13, startTime=1732032954857; duration=0sec 2024-11-19T16:15:54,916 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:54,916 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:B 2024-11-19T16:15:54,916 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:15:54,917 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:15:54,917 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/C is initiating minor compaction (all files) 2024-11-19T16:15:54,917 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/C in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:54,917 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/5b895b4fd2734696b98d567afbe7cefd, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/7650241d84be4749b1fef22638a49655, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/77c78d0f06a14a74979c8226eb2dfa3d] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=35.2 K 2024-11-19T16:15:54,917 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b895b4fd2734696b98d567afbe7cefd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732032951199 2024-11-19T16:15:54,917 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 7650241d84be4749b1fef22638a49655, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732032951292 2024-11-19T16:15:54,918 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 77c78d0f06a14a74979c8226eb2dfa3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032953492 2024-11-19T16:15:54,924 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#C#compaction#321 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:54,924 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/e9099603d6b4422b87faa5cea1695d17 is 50, key is test_row_0/C:col10/1732032953547/Put/seqid=0 2024-11-19T16:15:54,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742197_1373 (size=12104) 2024-11-19T16:15:54,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033014987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:54,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:54,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:55,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033014997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033014997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:55,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033015008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:55,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033015197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:55,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033015216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:55,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033015216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033015227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:55,295 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:55,299 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d4cc3820c716491fbbe7ecf65d64ae35_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d4cc3820c716491fbbe7ecf65d64ae35_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:55,301 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/7744607ec4f945e0a843b54fe6ff823b, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:55,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/7744607ec4f945e0a843b54fe6ff823b is 175, key is test_row_0/A:col10/1732032953656/Put/seqid=0 2024-11-19T16:15:55,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742198_1374 (size=39549) 2024-11-19T16:15:55,309 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/7744607ec4f945e0a843b54fe6ff823b 2024-11-19T16:15:55,317 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#A#compaction#319 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:55,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/01e3748b2a7845438923f0c796af2b78 is 50, key is test_row_0/B:col10/1732032953656/Put/seqid=0 2024-11-19T16:15:55,318 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/a7f593533d6342fcbb35a1d5960985d1 is 175, key is test_row_0/A:col10/1732032953547/Put/seqid=0 2024-11-19T16:15:55,332 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/e9099603d6b4422b87faa5cea1695d17 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/e9099603d6b4422b87faa5cea1695d17 2024-11-19T16:15:55,339 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/C of 47bdefb8e31bb07f134a6eace775a473 into e9099603d6b4422b87faa5cea1695d17(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:55,339 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:15:55,339 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/C, priority=13, startTime=1732032954857; duration=0sec 2024-11-19T16:15:55,339 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:55,339 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:C 2024-11-19T16:15:55,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742199_1375 (size=31058) 2024-11-19T16:15:55,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742200_1376 (size=12001) 2024-11-19T16:15:55,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/01e3748b2a7845438923f0c796af2b78 2024-11-19T16:15:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T16:15:55,357 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:55,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-19T16:15:55,360 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T16:15:55,360 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:55,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:55,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/8665ec1916a04135bce42c076f0e8e6c is 50, key is test_row_0/C:col10/1732032953656/Put/seqid=0 2024-11-19T16:15:55,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742201_1377 (size=12001) 2024-11-19T16:15:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T16:15:55,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T16:15:55,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:55,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:55,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:55,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:55,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:55,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033015518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:55,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033015527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033015527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:55,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:55,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033015546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:55,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T16:15:55,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:55,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:55,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:55,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:55,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T16:15:55,746 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/a7f593533d6342fcbb35a1d5960985d1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a7f593533d6342fcbb35a1d5960985d1 2024-11-19T16:15:55,750 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/A of 47bdefb8e31bb07f134a6eace775a473 into a7f593533d6342fcbb35a1d5960985d1(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:55,750 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:15:55,750 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/A, priority=13, startTime=1732032954855; duration=0sec 2024-11-19T16:15:55,750 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:55,750 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:A 2024-11-19T16:15:55,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/8665ec1916a04135bce42c076f0e8e6c 2024-11-19T16:15:55,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/7744607ec4f945e0a843b54fe6ff823b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7744607ec4f945e0a843b54fe6ff823b 2024-11-19T16:15:55,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7744607ec4f945e0a843b54fe6ff823b, entries=200, sequenceid=93, filesize=38.6 K 2024-11-19T16:15:55,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/01e3748b2a7845438923f0c796af2b78 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/01e3748b2a7845438923f0c796af2b78 2024-11-19T16:15:55,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/01e3748b2a7845438923f0c796af2b78, entries=150, sequenceid=93, filesize=11.7 K 2024-11-19T16:15:55,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/8665ec1916a04135bce42c076f0e8e6c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/8665ec1916a04135bce42c076f0e8e6c 2024-11-19T16:15:55,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/8665ec1916a04135bce42c076f0e8e6c, entries=150, sequenceid=93, filesize=11.7 K 2024-11-19T16:15:55,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for 47bdefb8e31bb07f134a6eace775a473 in 932ms, sequenceid=93, compaction requested=false 2024-11-19T16:15:55,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:15:55,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T16:15:55,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:55,822 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T16:15:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:15:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:15:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:15:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:55,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192540427fa4f940f6910abe4fe64f228a_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032954874/Put/seqid=0 2024-11-19T16:15:55,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742202_1378 (size=12154) 2024-11-19T16:15:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:55,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T16:15:55,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032955963, max timestamp=1732032955989Num uniques 859; 2024-11-19T16:15:55,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032955963, max timestamp=1732032955989Num uniques 859; 2024-11-19T16:15:55,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:15:55,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:15:55,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032955963, max timestamp=1732032955989Num uniques 859; 2024-11-19T16:15:55,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:15:56,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033016074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732032956064, max timestamp=1732032956086Num uniques 1018; 2024-11-19T16:15:56,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732032956064, max timestamp=1732032956086Num uniques 1018; 2024-11-19T16:15:56,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732032956064, max timestamp=1732032956086Num uniques 1018; 2024-11-19T16:15:56,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-19T16:15:56,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-19T16:15:56,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-19T16:15:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033016120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033016129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033016130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033016131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033016197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:56,245 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192540427fa4f940f6910abe4fe64f228a_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192540427fa4f940f6910abe4fe64f228a_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:56,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033016237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/bf83d25ef5ad4c8186e84f8cb238533b, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033016237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/bf83d25ef5ad4c8186e84f8cb238533b is 175, key is test_row_0/A:col10/1732032954874/Put/seqid=0 2024-11-19T16:15:56,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742203_1379 (size=30955) 2024-11-19T16:15:56,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033016257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033016257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033016416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033016457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033016457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033016466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033016466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T16:15:56,650 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/bf83d25ef5ad4c8186e84f8cb238533b 2024-11-19T16:15:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/78b792e4f1dc4ffda04a2ababe94a26f is 50, key is test_row_0/B:col10/1732032954874/Put/seqid=0 2024-11-19T16:15:56,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742204_1380 (size=12001) 2024-11-19T16:15:56,660 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/78b792e4f1dc4ffda04a2ababe94a26f 2024-11-19T16:15:56,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/d4d3e30eb1b94071ad913d54670b0a3e is 50, key is test_row_0/C:col10/1732032954874/Put/seqid=0 2024-11-19T16:15:56,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742205_1381 (size=12001) 2024-11-19T16:15:56,671 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/d4d3e30eb1b94071ad913d54670b0a3e 2024-11-19T16:15:56,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/bf83d25ef5ad4c8186e84f8cb238533b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/bf83d25ef5ad4c8186e84f8cb238533b 2024-11-19T16:15:56,678 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/bf83d25ef5ad4c8186e84f8cb238533b, entries=150, sequenceid=105, filesize=30.2 K 2024-11-19T16:15:56,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/78b792e4f1dc4ffda04a2ababe94a26f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/78b792e4f1dc4ffda04a2ababe94a26f 2024-11-19T16:15:56,682 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/78b792e4f1dc4ffda04a2ababe94a26f, entries=150, sequenceid=105, filesize=11.7 K 2024-11-19T16:15:56,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/d4d3e30eb1b94071ad913d54670b0a3e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/d4d3e30eb1b94071ad913d54670b0a3e 2024-11-19T16:15:56,685 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/d4d3e30eb1b94071ad913d54670b0a3e, entries=150, sequenceid=105, filesize=11.7 K 2024-11-19T16:15:56,686 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 47bdefb8e31bb07f134a6eace775a473 in 864ms, sequenceid=105, compaction requested=true 2024-11-19T16:15:56,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:15:56,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:56,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-19T16:15:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-19T16:15:56,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-19T16:15:56,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3270 sec 2024-11-19T16:15:56,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.3310 sec 2024-11-19T16:15:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:56,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=147.60 KB heapSize=414.61 KB 2024-11-19T16:15:56,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:15:56,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:56,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:15:56,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:56,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:15:56,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:56,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ed8f2b7265424b77bcca36b755de9e70_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032956731/Put/seqid=0 2024-11-19T16:15:56,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742206_1382 (size=14794) 2024-11-19T16:15:56,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033016777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033016777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033016786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033016787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033016787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:56,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033016907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:56,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:56,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:57,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033017118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:57,142 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:57,146 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ed8f2b7265424b77bcca36b755de9e70_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ed8f2b7265424b77bcca36b755de9e70_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:57,147 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/3004dd40a2024a09851d3e86530db3f6, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:57,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/3004dd40a2024a09851d3e86530db3f6 is 175, key is test_row_0/A:col10/1732032956731/Put/seqid=0 2024-11-19T16:15:57,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742207_1383 (size=39749) 2024-11-19T16:15:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033017297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033017297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:57,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033017307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033017307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:57,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033017436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T16:15:57,507 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:15:57,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:15:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-19T16:15:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T16:15:57,509 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:15:57,511 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:15:57,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:15:57,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/3004dd40a2024a09851d3e86530db3f6 2024-11-19T16:15:57,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/fda573e3f197453d9269c5f5f6eacbad is 50, key is test_row_0/B:col10/1732032956731/Put/seqid=0 2024-11-19T16:15:57,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742208_1384 (size=12151) 2024-11-19T16:15:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T16:15:57,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T16:15:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:57,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:57,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T16:15:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:57,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T16:15:57,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/fda573e3f197453d9269c5f5f6eacbad 2024-11-19T16:15:57,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:57,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033017957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:57,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T16:15:57,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:57,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:57,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:57,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:57,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:57,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:57,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/4c24ca1356ba4820987f12d3edefd9ad is 50, key is test_row_0/C:col10/1732032956731/Put/seqid=0 2024-11-19T16:15:57,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742209_1385 (size=12151) 2024-11-19T16:15:58,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T16:15:58,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:58,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:58,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:58,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:58,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T16:15:58,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T16:15:58,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:58,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:58,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:58,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:58,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:15:58,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033018317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:58,329 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 47bdefb8e31bb07f134a6eace775a473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:58,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033018327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:58,335 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 47bdefb8e31bb07f134a6eace775a473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:58,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033018327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:58,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033018327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:58,337 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 47bdefb8e31bb07f134a6eace775a473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:58,339 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 47bdefb8e31bb07f134a6eace775a473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:15:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:15:58,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/4c24ca1356ba4820987f12d3edefd9ad 2024-11-19T16:15:58,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/3004dd40a2024a09851d3e86530db3f6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3004dd40a2024a09851d3e86530db3f6 2024-11-19T16:15:58,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3004dd40a2024a09851d3e86530db3f6, entries=200, sequenceid=159, filesize=38.8 K 2024-11-19T16:15:58,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/fda573e3f197453d9269c5f5f6eacbad as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/fda573e3f197453d9269c5f5f6eacbad 2024-11-19T16:15:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/fda573e3f197453d9269c5f5f6eacbad, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T16:15:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/4c24ca1356ba4820987f12d3edefd9ad as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/4c24ca1356ba4820987f12d3edefd9ad 2024-11-19T16:15:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/4c24ca1356ba4820987f12d3edefd9ad, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T16:15:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~414.56 KB/424512, currentSize=33.54 KB/34350 for 47bdefb8e31bb07f134a6eace775a473 in 1678ms, sequenceid=159, compaction requested=true 2024-11-19T16:15:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:15:58,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:15:58,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:58,410 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:58,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:15:58,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:15:58,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T16:15:58,410 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,411 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:58,411 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141311 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:58,411 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/A is initiating minor compaction (all files) 2024-11-19T16:15:58,411 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/B is initiating minor compaction (all files) 2024-11-19T16:15:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,411 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/B in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:58,411 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/A in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:58,411 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a7f593533d6342fcbb35a1d5960985d1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7744607ec4f945e0a843b54fe6ff823b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/bf83d25ef5ad4c8186e84f8cb238533b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3004dd40a2024a09851d3e86530db3f6] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=138.0 K 2024-11-19T16:15:58,411 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/f66c3b363e2c4a129c0d1acb814ed4f2, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/01e3748b2a7845438923f0c796af2b78, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/78b792e4f1dc4ffda04a2ababe94a26f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/fda573e3f197453d9269c5f5f6eacbad] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=47.1 K 2024-11-19T16:15:58,412 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:58,412 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a7f593533d6342fcbb35a1d5960985d1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7744607ec4f945e0a843b54fe6ff823b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/bf83d25ef5ad4c8186e84f8cb238533b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3004dd40a2024a09851d3e86530db3f6] 2024-11-19T16:15:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,412 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting f66c3b363e2c4a129c0d1acb814ed4f2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032953492 2024-11-19T16:15:58,412 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7f593533d6342fcbb35a1d5960985d1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032953492 2024-11-19T16:15:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,413 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7744607ec4f945e0a843b54fe6ff823b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732032953656 2024-11-19T16:15:58,413 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 01e3748b2a7845438923f0c796af2b78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732032953656 2024-11-19T16:15:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,413 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf83d25ef5ad4c8186e84f8cb238533b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732032954867 2024-11-19T16:15:58,413 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 78b792e4f1dc4ffda04a2ababe94a26f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732032954867 2024-11-19T16:15:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,413 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting fda573e3f197453d9269c5f5f6eacbad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732032956128 2024-11-19T16:15:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,413 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3004dd40a2024a09851d3e86530db3f6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732032956116 2024-11-19T16:15:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,421 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,424 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#B#compaction#331 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,425 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/1eb2e6e92b0d49b782ee58df211789a6 is 50, key is test_row_0/B:col10/1732032956731/Put/seqid=0 2024-11-19T16:15:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,429 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411198111fbcbec414d37bea1b6e65388dd01_47bdefb8e31bb07f134a6eace775a473 store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T16:15:58,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:58,431 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T16:15:58,431 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411198111fbcbec414d37bea1b6e65388dd01_47bdefb8e31bb07f134a6eace775a473, store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:15:58,432 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198111fbcbec414d37bea1b6e65388dd01_47bdefb8e31bb07f134a6eace775a473 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:15:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742210_1386 (size=12391) 2024-11-19T16:15:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119075d79a89ffa4ac98ad83ca49e660c90_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_2/A:col10/1732032956784/Put/seqid=0 2024-11-19T16:15:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,454 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/1eb2e6e92b0d49b782ee58df211789a6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/1eb2e6e92b0d49b782ee58df211789a6 2024-11-19T16:15:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,458 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/B of 47bdefb8e31bb07f134a6eace775a473 into 1eb2e6e92b0d49b782ee58df211789a6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:58,458 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:15:58,459 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/B, priority=12, startTime=1732032958410; duration=0sec 2024-11-19T16:15:58,459 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:15:58,459 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:B 2024-11-19T16:15:58,459 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:15:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,460 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:15:58,460 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/C is initiating minor compaction (all files) 2024-11-19T16:15:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,460 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/C in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:58,460 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/e9099603d6b4422b87faa5cea1695d17, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/8665ec1916a04135bce42c076f0e8e6c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/d4d3e30eb1b94071ad913d54670b0a3e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/4c24ca1356ba4820987f12d3edefd9ad] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=47.1 K 2024-11-19T16:15:58,460 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting e9099603d6b4422b87faa5cea1695d17, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032953492 2024-11-19T16:15:58,460 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 8665ec1916a04135bce42c076f0e8e6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732032953656 2024-11-19T16:15:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,461 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting d4d3e30eb1b94071ad913d54670b0a3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732032954867 2024-11-19T16:15:58,461 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c24ca1356ba4820987f12d3edefd9ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732032956128 2024-11-19T16:15:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742211_1387 (size=4469) 2024-11-19T16:15:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,465 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#A#compaction#330 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,465 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/a907e34e756b4688b3b3efb95ee2671a is 175, key is test_row_0/A:col10/1732032956731/Put/seqid=0 2024-11-19T16:15:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,503 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#C#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:15:58,503 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/a3aee8ff491041179d45666919ce08bc is 50, key is test_row_0/C:col10/1732032956731/Put/seqid=0 2024-11-19T16:15:58,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742212_1388 (size=7324) 2024-11-19T16:15:58,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742213_1389 (size=31345) 2024-11-19T16:15:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742214_1390 (size=12391) 2024-11-19T16:15:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,513 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119075d79a89ffa4ac98ad83ca49e660c90_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119075d79a89ffa4ac98ad83ca49e660c90_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/023c1a4487094a09ac810d6d96d2dede, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/023c1a4487094a09ac810d6d96d2dede is 175, key is test_row_2/A:col10/1732032956784/Put/seqid=0 2024-11-19T16:15:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,516 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/a3aee8ff491041179d45666919ce08bc as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/a3aee8ff491041179d45666919ce08bc 2024-11-19T16:15:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,521 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/C of 47bdefb8e31bb07f134a6eace775a473 into a3aee8ff491041179d45666919ce08bc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:58,521 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:15:58,521 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/C, priority=12, startTime=1732032958410; duration=0sec 2024-11-19T16:15:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,521 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:58,521 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:C 2024-11-19T16:15:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742215_1391 (size=13815) 2024-11-19T16:15:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T16:15:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,916 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/a907e34e756b4688b3b3efb95ee2671a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a907e34e756b4688b3b3efb95ee2671a 2024-11-19T16:15:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,922 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/A of 47bdefb8e31bb07f134a6eace775a473 into a907e34e756b4688b3b3efb95ee2671a(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:15:58,922 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:15:58,922 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/A, priority=12, startTime=1732032958410; duration=0sec 2024-11-19T16:15:58,922 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:15:58,922 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:A 2024-11-19T16:15:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,955 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/023c1a4487094a09ac810d6d96d2dede 2024-11-19T16:15:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/6d3c740c2c4244cd906307fd3dd35270 is 50, key is test_row_2/B:col10/1732032956784/Put/seqid=0 2024-11-19T16:15:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742216_1392 (size=7365) 2024-11-19T16:15:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:59,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:15:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032959176, max timestamp=1732032959204Num uniques 859; 2024-11-19T16:15:59,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032959176, max timestamp=1732032959204Num uniques 859; 2024-11-19T16:15:59,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:59,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:59,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032959176, max timestamp=1732032959204Num uniques 859; 2024-11-19T16:15:59,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:15:59,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732032959274, max timestamp=1732032959339Num uniques 1068; 2024-11-19T16:15:59,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-19T16:15:59,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732032959274, max timestamp=1732032959339Num uniques 1068; 2024-11-19T16:15:59,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-19T16:15:59,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732032959274, max timestamp=1732032959339Num uniques 1068; 2024-11-19T16:15:59,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-19T16:15:59,407 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/6d3c740c2c4244cd906307fd3dd35270 2024-11-19T16:15:59,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/866e2a66279a492bbc0518a09165477e is 50, key is test_row_2/C:col10/1732032956784/Put/seqid=0 2024-11-19T16:15:59,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742217_1393 (size=7365) 2024-11-19T16:15:59,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732032959411, max timestamp=1732032959439Num uniques 1027; 2024-11-19T16:15:59,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T16:15:59,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732032959411, max timestamp=1732032959439Num uniques 1027; 2024-11-19T16:15:59,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T16:15:59,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732032959411, max timestamp=1732032959439Num uniques 1027; 2024-11-19T16:15:59,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T16:15:59,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732032959495, max timestamp=1732032959546Num uniques 1036; 2024-11-19T16:15:59,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732032959495, max timestamp=1732032959546Num uniques 1036; 2024-11-19T16:15:59,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-19T16:15:59,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-19T16:15:59,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732032959495, max timestamp=1732032959546Num uniques 1036; 2024-11-19T16:15:59,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-19T16:15:59,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T16:15:59,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732032959608, max timestamp=1732032959653Num uniques 1095; 2024-11-19T16:15:59,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732032959608, max timestamp=1732032959653Num uniques 1095; 2024-11-19T16:15:59,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-19T16:15:59,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-19T16:15:59,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732032959608, max timestamp=1732032959653Num uniques 1095; 2024-11-19T16:15:59,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-19T16:15:59,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732032959725, max timestamp=1732032959767Num uniques 1054; 2024-11-19T16:15:59,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732032959725, max timestamp=1732032959767Num uniques 1054; 2024-11-19T16:15:59,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732032959725, max timestamp=1732032959767Num uniques 1054; 2024-11-19T16:15:59,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-19T16:15:59,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-19T16:15:59,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-19T16:15:59,817 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/866e2a66279a492bbc0518a09165477e 2024-11-19T16:15:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/023c1a4487094a09ac810d6d96d2dede as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/023c1a4487094a09ac810d6d96d2dede 2024-11-19T16:15:59,824 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/023c1a4487094a09ac810d6d96d2dede, entries=50, sequenceid=167, filesize=13.5 K 2024-11-19T16:15:59,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/6d3c740c2c4244cd906307fd3dd35270 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6d3c740c2c4244cd906307fd3dd35270 2024-11-19T16:15:59,828 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6d3c740c2c4244cd906307fd3dd35270, entries=50, sequenceid=167, filesize=7.2 K 2024-11-19T16:15:59,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/866e2a66279a492bbc0518a09165477e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/866e2a66279a492bbc0518a09165477e 2024-11-19T16:15:59,832 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/866e2a66279a492bbc0518a09165477e, entries=50, sequenceid=167, filesize=7.2 K 2024-11-19T16:15:59,833 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=87.22 KB/89310 for 47bdefb8e31bb07f134a6eace775a473 in 1401ms, sequenceid=167, compaction requested=false 2024-11-19T16:15:59,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:15:59,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:15:59,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-19T16:15:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-19T16:15:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:59,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=93.93 KB heapSize=261.23 KB 2024-11-19T16:15:59,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:15:59,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:59,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:15:59,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:59,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:15:59,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:15:59,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-19T16:15:59,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3230 sec 2024-11-19T16:15:59,839 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.3310 sec 2024-11-19T16:15:59,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198b8691c9272d4dab9be9490533eb0f2e_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032959821/Put/seqid=0 2024-11-19T16:15:59,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742218_1394 (size=14994) 2024-11-19T16:15:59,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:15:59,853 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198b8691c9272d4dab9be9490533eb0f2e_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198b8691c9272d4dab9be9490533eb0f2e_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:15:59,854 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/b948b53e5ba5404fb250f887fc9a9bf8, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:15:59,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/b948b53e5ba5404fb250f887fc9a9bf8 is 175, key is test_row_0/A:col10/1732032959821/Put/seqid=0 2024-11-19T16:15:59,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742219_1395 (size=39949) 2024-11-19T16:15:59,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:15:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033019954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:15:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:15:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:00,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033020067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:00,257 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/b948b53e5ba5404fb250f887fc9a9bf8 2024-11-19T16:16:00,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/4047592c80b241c389618e4152a0a7c6 is 50, key is test_row_0/B:col10/1732032959821/Put/seqid=0 2024-11-19T16:16:00,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742220_1396 (size=12301) 2024-11-19T16:16:00,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033020293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:00,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033020346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:00,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033020346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:00,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033020346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:00,353 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 47bdefb8e31bb07f134a6eace775a473 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:00,353 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 47bdefb8e31bb07f134a6eace775a473 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:00,354 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 47bdefb8e31bb07f134a6eace775a473 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:00,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033020367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:00,380 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 47bdefb8e31bb07f134a6eace775a473 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:00,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033020607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:00,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/4047592c80b241c389618e4152a0a7c6 2024-11-19T16:16:00,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/0ce23a1883f647f1ac91a456bb0c6183 is 50, key is test_row_0/C:col10/1732032959821/Put/seqid=0 2024-11-19T16:16:00,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742221_1397 (size=12301) 2024-11-19T16:16:01,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/0ce23a1883f647f1ac91a456bb0c6183 2024-11-19T16:16:01,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/b948b53e5ba5404fb250f887fc9a9bf8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/b948b53e5ba5404fb250f887fc9a9bf8 2024-11-19T16:16:01,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/b948b53e5ba5404fb250f887fc9a9bf8, entries=200, sequenceid=286, filesize=39.0 K 2024-11-19T16:16:01,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/4047592c80b241c389618e4152a0a7c6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/4047592c80b241c389618e4152a0a7c6 2024-11-19T16:16:01,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/4047592c80b241c389618e4152a0a7c6, entries=150, sequenceid=286, filesize=12.0 K 2024-11-19T16:16:01,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/0ce23a1883f647f1ac91a456bb0c6183 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/0ce23a1883f647f1ac91a456bb0c6183 2024-11-19T16:16:01,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/0ce23a1883f647f1ac91a456bb0c6183, entries=150, sequenceid=286, filesize=12.0 K 2024-11-19T16:16:01,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~261.19 KB/267456, currentSize=87.22 KB/89310 for 47bdefb8e31bb07f134a6eace775a473 in 1265ms, sequenceid=286, compaction requested=true 2024-11-19T16:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:01,099 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:01,099 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:01,100 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32057 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:01,100 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:01,100 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/B is initiating minor compaction (all files) 2024-11-19T16:16:01,100 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/A is initiating minor compaction (all files) 2024-11-19T16:16:01,100 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/B in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:01,100 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/A in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:01,100 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/1eb2e6e92b0d49b782ee58df211789a6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6d3c740c2c4244cd906307fd3dd35270, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/4047592c80b241c389618e4152a0a7c6] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=31.3 K 2024-11-19T16:16:01,100 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a907e34e756b4688b3b3efb95ee2671a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/023c1a4487094a09ac810d6d96d2dede, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/b948b53e5ba5404fb250f887fc9a9bf8] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=83.1 K 2024-11-19T16:16:01,100 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:01,100 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a907e34e756b4688b3b3efb95ee2671a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/023c1a4487094a09ac810d6d96d2dede, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/b948b53e5ba5404fb250f887fc9a9bf8] 2024-11-19T16:16:01,100 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eb2e6e92b0d49b782ee58df211789a6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732032956128 2024-11-19T16:16:01,101 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting a907e34e756b4688b3b3efb95ee2671a, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732032956128 2024-11-19T16:16:01,101 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d3c740c2c4244cd906307fd3dd35270, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732032956784 2024-11-19T16:16:01,101 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 023c1a4487094a09ac810d6d96d2dede, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732032956784 2024-11-19T16:16:01,101 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 4047592c80b241c389618e4152a0a7c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732032959797 2024-11-19T16:16:01,101 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting b948b53e5ba5404fb250f887fc9a9bf8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732032959797 2024-11-19T16:16:01,108 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:01,110 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119740839f13aca4d219b7c296a176e7947_47bdefb8e31bb07f134a6eace775a473 store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:01,110 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#B#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:01,110 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/9207fa66eeb04d1ab7d3df6f7e6c3479 is 50, key is test_row_0/B:col10/1732032959821/Put/seqid=0 2024-11-19T16:16:01,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742222_1398 (size=12643) 2024-11-19T16:16:01,117 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119740839f13aca4d219b7c296a176e7947_47bdefb8e31bb07f134a6eace775a473, store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:01,117 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119740839f13aca4d219b7c296a176e7947_47bdefb8e31bb07f134a6eace775a473 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:01,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742223_1399 (size=4469) 2024-11-19T16:16:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:01,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T16:16:01,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:16:01,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:01,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:16:01,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:01,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:16:01,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:01,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190e4462f03b3d4ba99070c17d7543f480_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032959897/Put/seqid=0 2024-11-19T16:16:01,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742224_1400 (size=14994) 2024-11-19T16:16:01,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033021210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:01,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033021327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:01,520 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/9207fa66eeb04d1ab7d3df6f7e6c3479 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/9207fa66eeb04d1ab7d3df6f7e6c3479 2024-11-19T16:16:01,524 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#A#compaction#339 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:01,524 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/0f42c473a8a242b6a99041668b9a308c is 175, key is test_row_0/A:col10/1732032959821/Put/seqid=0 2024-11-19T16:16:01,525 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/B of 47bdefb8e31bb07f134a6eace775a473 into 9207fa66eeb04d1ab7d3df6f7e6c3479(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:01,525 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:01,525 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/B, priority=13, startTime=1732032961099; duration=0sec 2024-11-19T16:16:01,525 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:01,525 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:B 2024-11-19T16:16:01,525 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:01,526 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32057 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:01,527 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/C is initiating minor compaction (all files) 2024-11-19T16:16:01,527 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/C in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:01,527 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/a3aee8ff491041179d45666919ce08bc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/866e2a66279a492bbc0518a09165477e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/0ce23a1883f647f1ac91a456bb0c6183] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=31.3 K 2024-11-19T16:16:01,527 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a3aee8ff491041179d45666919ce08bc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732032956128 2024-11-19T16:16:01,527 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 866e2a66279a492bbc0518a09165477e, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732032956784 2024-11-19T16:16:01,527 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ce23a1883f647f1ac91a456bb0c6183, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732032959797 2024-11-19T16:16:01,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742225_1401 (size=31597) 2024-11-19T16:16:01,535 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/0f42c473a8a242b6a99041668b9a308c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/0f42c473a8a242b6a99041668b9a308c 2024-11-19T16:16:01,537 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:01,537 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/c99038426115469a8581470e13af25a9 is 50, key is test_row_0/C:col10/1732032959821/Put/seqid=0 2024-11-19T16:16:01,543 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/A of 47bdefb8e31bb07f134a6eace775a473 into 0f42c473a8a242b6a99041668b9a308c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:01,543 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:01,543 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/A, priority=13, startTime=1732032961099; duration=0sec 2024-11-19T16:16:01,543 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:01,543 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:A 2024-11-19T16:16:01,546 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:01,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742226_1402 (size=12643) 2024-11-19T16:16:01,551 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190e4462f03b3d4ba99070c17d7543f480_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190e4462f03b3d4ba99070c17d7543f480_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:01,551 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/e8a2f7b2467444a28a652b93353ffb3c, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:01,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/e8a2f7b2467444a28a652b93353ffb3c is 175, key is test_row_0/A:col10/1732032959897/Put/seqid=0 2024-11-19T16:16:01,553 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/c99038426115469a8581470e13af25a9 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/c99038426115469a8581470e13af25a9 2024-11-19T16:16:01,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742227_1403 (size=39949) 2024-11-19T16:16:01,556 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/e8a2f7b2467444a28a652b93353ffb3c 2024-11-19T16:16:01,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033021547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:01,561 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/C of 47bdefb8e31bb07f134a6eace775a473 into c99038426115469a8581470e13af25a9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:01,561 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:01,561 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/C, priority=13, startTime=1732032961099; duration=0sec 2024-11-19T16:16:01,561 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:01,561 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:C 2024-11-19T16:16:01,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/0592519292f44fbfa5263a15732b66b2 is 50, key is test_row_0/B:col10/1732032959897/Put/seqid=0 2024-11-19T16:16:01,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742228_1404 (size=12301) 2024-11-19T16:16:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T16:16:01,677 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:01,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-19T16:16:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T16:16:01,679 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:01,680 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:01,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T16:16:01,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T16:16:01,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:01,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:01,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:01,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:01,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:01,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033021867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:01,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/0592519292f44fbfa5263a15732b66b2 2024-11-19T16:16:01,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/37ec1ac5537746eaa9da0f37dba60451 is 50, key is test_row_0/C:col10/1732032959897/Put/seqid=0 2024-11-19T16:16:01,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T16:16:01,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742229_1405 (size=12301) 2024-11-19T16:16:01,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:01,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:01,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:01,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:01,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T16:16:02,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T16:16:02,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:02,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:02,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:02,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:02,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:02,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T16:16:02,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:02,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:02,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:02,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:02,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T16:16:02,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/37ec1ac5537746eaa9da0f37dba60451 2024-11-19T16:16:02,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/e8a2f7b2467444a28a652b93353ffb3c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/e8a2f7b2467444a28a652b93353ffb3c 2024-11-19T16:16:02,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/e8a2f7b2467444a28a652b93353ffb3c, entries=200, sequenceid=303, filesize=39.0 K 2024-11-19T16:16:02,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/0592519292f44fbfa5263a15732b66b2 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/0592519292f44fbfa5263a15732b66b2 2024-11-19T16:16:02,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033022387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:02,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:02,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:02,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:02,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/0592519292f44fbfa5263a15732b66b2, entries=150, sequenceid=303, filesize=12.0 K 2024-11-19T16:16:02,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/37ec1ac5537746eaa9da0f37dba60451 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/37ec1ac5537746eaa9da0f37dba60451 2024-11-19T16:16:02,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/37ec1ac5537746eaa9da0f37dba60451, entries=150, sequenceid=303, filesize=12.0 K 2024-11-19T16:16:02,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 47bdefb8e31bb07f134a6eace775a473 in 1274ms, sequenceid=303, compaction requested=false 2024-11-19T16:16:02,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:02,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T16:16:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:02,443 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T16:16:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:16:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:16:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:16:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:02,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b086ee0b6689482c88142d6e634a9d32_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032961167/Put/seqid=0 2024-11-19T16:16:02,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742230_1406 (size=12454) 2024-11-19T16:16:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T16:16:02,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:02,879 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b086ee0b6689482c88142d6e634a9d32_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b086ee0b6689482c88142d6e634a9d32_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:02,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/d2ab716494cf4842b1f36d0ed3dff931, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:02,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/d2ab716494cf4842b1f36d0ed3dff931 is 175, key is test_row_0/A:col10/1732032961167/Put/seqid=0 2024-11-19T16:16:02,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742231_1407 (size=31255) 2024-11-19T16:16:02,885 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/d2ab716494cf4842b1f36d0ed3dff931 2024-11-19T16:16:02,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/a0505f8001e246da8c8907b32fef4248 is 50, key is test_row_0/B:col10/1732032961167/Put/seqid=0 2024-11-19T16:16:02,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742232_1408 (size=12301) 2024-11-19T16:16:03,296 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/a0505f8001e246da8c8907b32fef4248 2024-11-19T16:16:03,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/cd7f516f5ee24afaa14b9d06e7031a46 is 50, key is test_row_0/C:col10/1732032961167/Put/seqid=0 2024-11-19T16:16:03,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742233_1409 (size=12301) 2024-11-19T16:16:03,363 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/cd7f516f5ee24afaa14b9d06e7031a46 2024-11-19T16:16:03,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/d2ab716494cf4842b1f36d0ed3dff931 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/d2ab716494cf4842b1f36d0ed3dff931 2024-11-19T16:16:03,385 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/d2ab716494cf4842b1f36d0ed3dff931, entries=150, sequenceid=321, filesize=30.5 K 2024-11-19T16:16:03,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T16:16:03,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/a0505f8001e246da8c8907b32fef4248 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a0505f8001e246da8c8907b32fef4248 2024-11-19T16:16:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,390 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a0505f8001e246da8c8907b32fef4248, entries=150, sequenceid=321, filesize=12.0 K 2024-11-19T16:16:03,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/cd7f516f5ee24afaa14b9d06e7031a46 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/cd7f516f5ee24afaa14b9d06e7031a46 2024-11-19T16:16:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,397 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/cd7f516f5ee24afaa14b9d06e7031a46, entries=150, sequenceid=321, filesize=12.0 K 2024-11-19T16:16:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,398 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 47bdefb8e31bb07f134a6eace775a473 in 955ms, sequenceid=321, compaction requested=true 2024-11-19T16:16:03,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:03,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:03,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-19T16:16:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-19T16:16:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-19T16:16:03,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7200 sec 2024-11-19T16:16:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.7270 sec 2024-11-19T16:16:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T16:16:03,827 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,828 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-19T16:16:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,831 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,832 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:03,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T16:16:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:03,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:16:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:16:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:16:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:16:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:03,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196e32cebf5041466382fe0a713dcb56d9_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032963885/Put/seqid=0 2024-11-19T16:16:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T16:16:03,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742234_1410 (size=17534) 2024-11-19T16:16:03,974 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:03,978 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196e32cebf5041466382fe0a713dcb56d9_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196e32cebf5041466382fe0a713dcb56d9_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:03,979 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/2370f6a4d6f54d6ca4d8b61112c572fe, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:03,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/2370f6a4d6f54d6ca4d8b61112c572fe is 175, key is test_row_0/A:col10/1732032963885/Put/seqid=0 2024-11-19T16:16:03,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T16:16:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:03,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:03,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742235_1411 (size=48639) 2024-11-19T16:16:04,010 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/2370f6a4d6f54d6ca4d8b61112c572fe 2024-11-19T16:16:04,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/6f18595d1a2d41d7bfbe1b41b1cb429b is 50, key is test_row_0/B:col10/1732032963885/Put/seqid=0 2024-11-19T16:16:04,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742236_1412 (size=12301) 2024-11-19T16:16:04,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/6f18595d1a2d41d7bfbe1b41b1cb429b 2024-11-19T16:16:04,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/a52bde9f23d64737bc968c17d75a0062 is 50, key is test_row_0/C:col10/1732032963885/Put/seqid=0 2024-11-19T16:16:04,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742237_1413 (size=12301) 2024-11-19T16:16:04,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T16:16:04,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:04,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T16:16:04,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032964077, max timestamp=1732032964222Num uniques 859; 2024-11-19T16:16:04,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:04,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032964077, max timestamp=1732032964222Num uniques 859; 2024-11-19T16:16:04,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032964077, max timestamp=1732032964222Num uniques 859; 2024-11-19T16:16:04,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:04,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:04,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T16:16:04,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:04,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40604 deadline: 1732033024407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,417 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 47bdefb8e31bb07f134a6eace775a473 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033024408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033024413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033024415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033024417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T16:16:04,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T16:16:04,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:04,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/a52bde9f23d64737bc968c17d75a0062 2024-11-19T16:16:04,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/2370f6a4d6f54d6ca4d8b61112c572fe as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/2370f6a4d6f54d6ca4d8b61112c572fe 2024-11-19T16:16:04,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/2370f6a4d6f54d6ca4d8b61112c572fe, entries=250, sequenceid=331, filesize=47.5 K 2024-11-19T16:16:04,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/6f18595d1a2d41d7bfbe1b41b1cb429b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6f18595d1a2d41d7bfbe1b41b1cb429b 2024-11-19T16:16:04,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6f18595d1a2d41d7bfbe1b41b1cb429b, entries=150, sequenceid=331, filesize=12.0 K 2024-11-19T16:16:04,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/a52bde9f23d64737bc968c17d75a0062 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/a52bde9f23d64737bc968c17d75a0062 2024-11-19T16:16:04,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/a52bde9f23d64737bc968c17d75a0062, entries=150, sequenceid=331, filesize=12.0 K 2024-11-19T16:16:04,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 47bdefb8e31bb07f134a6eace775a473 in 638ms, sequenceid=331, compaction requested=true 2024-11-19T16:16:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:04,530 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:04,530 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:04,532 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:04,532 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151440 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:04,532 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/A is initiating minor compaction (all files) 2024-11-19T16:16:04,532 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/B is initiating minor compaction (all files) 2024-11-19T16:16:04,532 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/A in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,532 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/0f42c473a8a242b6a99041668b9a308c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/e8a2f7b2467444a28a652b93353ffb3c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/d2ab716494cf4842b1f36d0ed3dff931, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/2370f6a4d6f54d6ca4d8b61112c572fe] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=147.9 K 2024-11-19T16:16:04,532 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,532 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/0f42c473a8a242b6a99041668b9a308c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/e8a2f7b2467444a28a652b93353ffb3c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/d2ab716494cf4842b1f36d0ed3dff931, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/2370f6a4d6f54d6ca4d8b61112c572fe] 2024-11-19T16:16:04,532 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/B in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,532 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/9207fa66eeb04d1ab7d3df6f7e6c3479, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/0592519292f44fbfa5263a15732b66b2, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a0505f8001e246da8c8907b32fef4248, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6f18595d1a2d41d7bfbe1b41b1cb429b] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=48.4 K 2024-11-19T16:16:04,533 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 9207fa66eeb04d1ab7d3df6f7e6c3479, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732032959797 2024-11-19T16:16:04,533 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f42c473a8a242b6a99041668b9a308c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732032959797 2024-11-19T16:16:04,533 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 0592519292f44fbfa5263a15732b66b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732032959897 2024-11-19T16:16:04,534 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8a2f7b2467444a28a652b93353ffb3c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732032959897 2024-11-19T16:16:04,535 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2ab716494cf4842b1f36d0ed3dff931, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732032961167 2024-11-19T16:16:04,535 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a0505f8001e246da8c8907b32fef4248, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732032961167 2024-11-19T16:16:04,535 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f18595d1a2d41d7bfbe1b41b1cb429b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732032963791 2024-11-19T16:16:04,535 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2370f6a4d6f54d6ca4d8b61112c572fe, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732032963791 2024-11-19T16:16:04,546 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:04,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-19T16:16:04,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:16:04,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:04,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:16:04,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:04,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:16:04,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:04,549 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119942b417fa01349278892667adc37ccca_47bdefb8e31bb07f134a6eace775a473 store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:04,550 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#B#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:04,551 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/0509d32c673649b891c7f41db6c4b52f is 50, key is test_row_0/B:col10/1732032963885/Put/seqid=0 2024-11-19T16:16:04,552 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119942b417fa01349278892667adc37ccca_47bdefb8e31bb07f134a6eace775a473, store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:04,552 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119942b417fa01349278892667adc37ccca_47bdefb8e31bb07f134a6eace775a473 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:04,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192cb160522142456aa5e6032a84e74a55_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032964543/Put/seqid=0 2024-11-19T16:16:04,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033024570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742238_1414 (size=12779) 2024-11-19T16:16:04,579 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/0509d32c673649b891c7f41db6c4b52f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/0509d32c673649b891c7f41db6c4b52f 2024-11-19T16:16:04,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742239_1415 (size=4469) 2024-11-19T16:16:04,584 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/B of 47bdefb8e31bb07f134a6eace775a473 into 0509d32c673649b891c7f41db6c4b52f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:04,584 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:04,584 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/B, priority=12, startTime=1732032964530; duration=0sec 2024-11-19T16:16:04,584 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:04,584 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:B 2024-11-19T16:16:04,584 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:04,585 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:04,585 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/C is initiating minor compaction (all files) 2024-11-19T16:16:04,585 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/C in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,585 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/c99038426115469a8581470e13af25a9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/37ec1ac5537746eaa9da0f37dba60451, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/cd7f516f5ee24afaa14b9d06e7031a46, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/a52bde9f23d64737bc968c17d75a0062] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=48.4 K 2024-11-19T16:16:04,586 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting c99038426115469a8581470e13af25a9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732032959797 2024-11-19T16:16:04,586 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 37ec1ac5537746eaa9da0f37dba60451, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732032959897 2024-11-19T16:16:04,586 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting cd7f516f5ee24afaa14b9d06e7031a46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732032961167 2024-11-19T16:16:04,586 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a52bde9f23d64737bc968c17d75a0062, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732032963791 2024-11-19T16:16:04,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033024572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033024572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033024572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,603 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#A#compaction#351 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:04,603 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/c41bf49794d14d37b7f75ee2f0a7ebd2 is 175, key is test_row_0/A:col10/1732032963885/Put/seqid=0 2024-11-19T16:16:04,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742240_1416 (size=14994) 2024-11-19T16:16:04,604 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:04,608 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192cb160522142456aa5e6032a84e74a55_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192cb160522142456aa5e6032a84e74a55_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:04,608 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#C#compaction#354 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:04,609 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/3d2b267211f245398526cece7c8d9154, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:04,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/3d2b267211f245398526cece7c8d9154 is 175, key is test_row_0/A:col10/1732032964543/Put/seqid=0 2024-11-19T16:16:04,612 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/3b378a7db3e94b39b7b8fbb32c6b217f is 50, key is test_row_0/C:col10/1732032963885/Put/seqid=0 2024-11-19T16:16:04,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T16:16:04,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742241_1417 (size=31733) 2024-11-19T16:16:04,653 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/c41bf49794d14d37b7f75ee2f0a7ebd2 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/c41bf49794d14d37b7f75ee2f0a7ebd2 2024-11-19T16:16:04,658 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/A of 47bdefb8e31bb07f134a6eace775a473 into c41bf49794d14d37b7f75ee2f0a7ebd2(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:04,658 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:04,658 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/A, priority=12, startTime=1732032964530; duration=0sec 2024-11-19T16:16:04,658 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:04,658 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:A 2024-11-19T16:16:04,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742242_1418 (size=39949) 2024-11-19T16:16:04,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742243_1419 (size=12779) 2024-11-19T16:16:04,679 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/3d2b267211f245398526cece7c8d9154 2024-11-19T16:16:04,684 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/3b378a7db3e94b39b7b8fbb32c6b217f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/3b378a7db3e94b39b7b8fbb32c6b217f 2024-11-19T16:16:04,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/62a5655eed264913b5183a59f1b6fc4b is 50, key is test_row_0/B:col10/1732032964543/Put/seqid=0 2024-11-19T16:16:04,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033024677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,697 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/C of 47bdefb8e31bb07f134a6eace775a473 into 3b378a7db3e94b39b7b8fbb32c6b217f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:04,697 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:04,697 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/C, priority=12, startTime=1732032964530; duration=0sec 2024-11-19T16:16:04,697 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:04,697 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:C 2024-11-19T16:16:04,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033024697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033024697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033024697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742244_1420 (size=12301) 2024-11-19T16:16:04,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T16:16:04,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:04,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033024897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T16:16:04,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:04,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:04,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:04,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033024917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033024917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033024917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:04,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T16:16:05,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T16:16:05,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:05,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:05,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:05,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:05,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:05,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/62a5655eed264913b5183a59f1b6fc4b 2024-11-19T16:16:05,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/d00644bbb9ca4a058e5d899e2f5d6a0a is 50, key is test_row_0/C:col10/1732032964543/Put/seqid=0 2024-11-19T16:16:05,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742245_1421 (size=12301) 2024-11-19T16:16:05,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033025220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:05,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033025249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:05,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033025249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:05,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033025250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:05,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T16:16:05,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:05,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:05,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:05,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:05,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:05,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T16:16:05,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:05,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:05,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:05,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T16:16:05,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:05,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:05,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:05,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:05,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:05,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/d00644bbb9ca4a058e5d899e2f5d6a0a 2024-11-19T16:16:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:05,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/3d2b267211f245398526cece7c8d9154 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3d2b267211f245398526cece7c8d9154 2024-11-19T16:16:05,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3d2b267211f245398526cece7c8d9154, entries=200, sequenceid=368, filesize=39.0 K 2024-11-19T16:16:05,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/62a5655eed264913b5183a59f1b6fc4b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/62a5655eed264913b5183a59f1b6fc4b 2024-11-19T16:16:05,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/62a5655eed264913b5183a59f1b6fc4b, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T16:16:05,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/d00644bbb9ca4a058e5d899e2f5d6a0a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/d00644bbb9ca4a058e5d899e2f5d6a0a 2024-11-19T16:16:05,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/d00644bbb9ca4a058e5d899e2f5d6a0a, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T16:16:05,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 47bdefb8e31bb07f134a6eace775a473 in 1062ms, sequenceid=368, compaction requested=false 2024-11-19T16:16:05,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:05,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T16:16:05,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:05,742 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T16:16:05,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:05,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:16:05,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:05,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:16:05,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:05,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:16:05,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:05,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119237256b42cec412b85fc67252dac4088_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032965740/Put/seqid=0 2024-11-19T16:16:05,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742246_1422 (size=14994) 2024-11-19T16:16:05,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032965813, max timestamp=1732032965837Num uniques 859; 2024-11-19T16:16:05,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:16:05,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032965813, max timestamp=1732032965837Num uniques 859; 2024-11-19T16:16:05,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732032965813, max timestamp=1732032965837Num uniques 859; 2024-11-19T16:16:05,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:16:05,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:16:05,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033025907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:05,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:05,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033025915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033025915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:05,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033025929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T16:16:06,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033026037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:06,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033026037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:06,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033026037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:06,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033026039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:06,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:06,209 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119237256b42cec412b85fc67252dac4088_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119237256b42cec412b85fc67252dac4088_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/85ac58f370ad492ba6d3a3dd806b0dd9, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/85ac58f370ad492ba6d3a3dd806b0dd9 is 175, key is test_row_0/A:col10/1732032965740/Put/seqid=0 2024-11-19T16:16:06,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742247_1423 (size=39949) 2024-11-19T16:16:06,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033026257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:06,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033026257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:06,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033026267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:06,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033026267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:06,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033026567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033026567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:06,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033026587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:06,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:06,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033026587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:06,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:06,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:06,626 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=381, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/85ac58f370ad492ba6d3a3dd806b0dd9 2024-11-19T16:16:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/080f8e8ad46849b89971e4eddf8f6479 is 50, key is test_row_0/B:col10/1732032965740/Put/seqid=0 2024-11-19T16:16:06,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742248_1424 (size=12301) 2024-11-19T16:16:07,035 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/080f8e8ad46849b89971e4eddf8f6479 2024-11-19T16:16:07,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/02970c88db274868bd4d3473923d48c8 is 50, key is test_row_0/C:col10/1732032965740/Put/seqid=0 2024-11-19T16:16:07,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742249_1425 (size=12301) 2024-11-19T16:16:07,074 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/02970c88db274868bd4d3473923d48c8 2024-11-19T16:16:07,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/85ac58f370ad492ba6d3a3dd806b0dd9 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/85ac58f370ad492ba6d3a3dd806b0dd9 2024-11-19T16:16:07,088 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/85ac58f370ad492ba6d3a3dd806b0dd9, entries=200, sequenceid=381, filesize=39.0 K 2024-11-19T16:16:07,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/080f8e8ad46849b89971e4eddf8f6479 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/080f8e8ad46849b89971e4eddf8f6479 2024-11-19T16:16:07,100 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/080f8e8ad46849b89971e4eddf8f6479, entries=150, sequenceid=381, filesize=12.0 K 2024-11-19T16:16:07,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033027088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:07,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033027088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/02970c88db274868bd4d3473923d48c8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/02970c88db274868bd4d3473923d48c8 2024-11-19T16:16:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:07,107 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/02970c88db274868bd4d3473923d48c8, entries=150, sequenceid=381, filesize=12.0 K 2024-11-19T16:16:07,108 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 47bdefb8e31bb07f134a6eace775a473 in 1366ms, sequenceid=381, compaction requested=true 2024-11-19T16:16:07,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:07,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:07,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-19T16:16:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-19T16:16:07,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-19T16:16:07,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2810 sec 2024-11-19T16:16:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:07,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732032965910, max timestamp=1732032967119Num uniques 1018; 2024-11-19T16:16:07,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T16:16:07,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732032965910, max timestamp=1732032967119Num uniques 1018; 2024-11-19T16:16:07,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T16:16:07,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=71.56 KB heapSize=198.72 KB 2024-11-19T16:16:07,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:16:07,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:07,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:16:07,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:07,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:16:07,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T16:16:07,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.2990 sec 2024-11-19T16:16:07,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111988b5e57a5e0a48faa41711d9c7841f7d_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032967119/Put/seqid=0 2024-11-19T16:16:07,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742250_1426 (size=17534) 2024-11-19T16:16:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033027205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:07,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033027206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:07,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033027327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:07,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:07,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033027327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:07,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:07,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:07,545 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:07,547 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111988b5e57a5e0a48faa41711d9c7841f7d_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111988b5e57a5e0a48faa41711d9c7841f7d_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:07,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033027536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:07,548 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/6cd40f7f96934859838fd11d48d74d28, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:07,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:07,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033027536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:07,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/6cd40f7f96934859838fd11d48d74d28 is 175, key is test_row_0/A:col10/1732032967119/Put/seqid=0 2024-11-19T16:16:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:07,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742251_1427 (size=48639) 2024-11-19T16:16:07,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033027857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:07,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033027857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:07,952 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=420, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/6cd40f7f96934859838fd11d48d74d28 2024-11-19T16:16:07,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/b3726a0a8a8b4399b56a7dae6bc9cf01 is 50, key is test_row_0/B:col10/1732032967119/Put/seqid=0 2024-11-19T16:16:07,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742252_1428 (size=12301) 2024-11-19T16:16:08,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T16:16:08,007 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:08,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-19T16:16:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T16:16:08,009 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:08,010 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:08,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T16:16:08,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033028107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:08,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033028116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:08,119 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 47bdefb8e31bb07f134a6eace775a473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:08,119 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 47bdefb8e31bb07f134a6eace775a473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:08,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T16:16:08,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:08,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T16:16:08,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T16:16:08,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/b3726a0a8a8b4399b56a7dae6bc9cf01 2024-11-19T16:16:08,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/e918afab612f46f390b4280015e2d63b is 50, key is test_row_0/C:col10/1732032967119/Put/seqid=0 2024-11-19T16:16:08,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033028377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:08,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033028377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:08,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742253_1429 (size=12301) 2024-11-19T16:16:08,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T16:16:08,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:08,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T16:16:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T16:16:08,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T16:16:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:08,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/e918afab612f46f390b4280015e2d63b 2024-11-19T16:16:08,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/6cd40f7f96934859838fd11d48d74d28 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6cd40f7f96934859838fd11d48d74d28 2024-11-19T16:16:08,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6cd40f7f96934859838fd11d48d74d28, entries=250, sequenceid=420, filesize=47.5 K 2024-11-19T16:16:08,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/b3726a0a8a8b4399b56a7dae6bc9cf01 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/b3726a0a8a8b4399b56a7dae6bc9cf01 2024-11-19T16:16:08,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/b3726a0a8a8b4399b56a7dae6bc9cf01, entries=150, sequenceid=420, filesize=12.0 K 2024-11-19T16:16:08,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/e918afab612f46f390b4280015e2d63b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/e918afab612f46f390b4280015e2d63b 2024-11-19T16:16:08,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/e918afab612f46f390b4280015e2d63b, entries=150, sequenceid=420, filesize=12.0 K 2024-11-19T16:16:08,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~198.67 KB/203440, currentSize=107.34 KB/109920 for 47bdefb8e31bb07f134a6eace775a473 in 1741ms, sequenceid=420, compaction requested=true 2024-11-19T16:16:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:08,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:08,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:08,868 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:08,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:08,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:08,868 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:08,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:08,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:08,868 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:08,868 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160270 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:08,868 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/A is initiating minor compaction (all files) 2024-11-19T16:16:08,868 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/B is initiating minor compaction (all files) 2024-11-19T16:16:08,869 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/B in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,869 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/A in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,869 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/0509d32c673649b891c7f41db6c4b52f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/62a5655eed264913b5183a59f1b6fc4b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/080f8e8ad46849b89971e4eddf8f6479, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/b3726a0a8a8b4399b56a7dae6bc9cf01] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=48.5 K 2024-11-19T16:16:08,869 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/c41bf49794d14d37b7f75ee2f0a7ebd2, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3d2b267211f245398526cece7c8d9154, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/85ac58f370ad492ba6d3a3dd806b0dd9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6cd40f7f96934859838fd11d48d74d28] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=156.5 K 2024-11-19T16:16:08,869 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,869 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/c41bf49794d14d37b7f75ee2f0a7ebd2, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3d2b267211f245398526cece7c8d9154, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/85ac58f370ad492ba6d3a3dd806b0dd9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6cd40f7f96934859838fd11d48d74d28] 2024-11-19T16:16:08,869 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 0509d32c673649b891c7f41db6c4b52f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732032963791 2024-11-19T16:16:08,869 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting c41bf49794d14d37b7f75ee2f0a7ebd2, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732032963791 2024-11-19T16:16:08,869 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 62a5655eed264913b5183a59f1b6fc4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732032964411 2024-11-19T16:16:08,869 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d2b267211f245398526cece7c8d9154, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732032964411 2024-11-19T16:16:08,869 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 080f8e8ad46849b89971e4eddf8f6479, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732032964549 2024-11-19T16:16:08,870 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85ac58f370ad492ba6d3a3dd806b0dd9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732032964549 2024-11-19T16:16:08,870 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting b3726a0a8a8b4399b56a7dae6bc9cf01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732032965926 2024-11-19T16:16:08,870 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cd40f7f96934859838fd11d48d74d28, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732032965910 2024-11-19T16:16:08,875 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:08,877 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:08,878 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/bc87d4ec5bcf41068bdfb8db7b05a100 is 50, key is test_row_0/B:col10/1732032967119/Put/seqid=0 2024-11-19T16:16:08,881 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119b6c03975010048499f6be9187922481c_47bdefb8e31bb07f134a6eace775a473 store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:08,883 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119b6c03975010048499f6be9187922481c_47bdefb8e31bb07f134a6eace775a473, store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:08,884 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b6c03975010048499f6be9187922481c_47bdefb8e31bb07f134a6eace775a473 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:08,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742254_1430 (size=12915) 2024-11-19T16:16:08,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742255_1431 (size=4469) 2024-11-19T16:16:08,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T16:16:08,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:08,929 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T16:16:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:16:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:16:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:16:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:08,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f3d4b59d5c67477bb764acda4350adb9_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032967204/Put/seqid=0 2024-11-19T16:16:08,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742256_1432 (size=12454) 2024-11-19T16:16:08,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:08,947 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f3d4b59d5c67477bb764acda4350adb9_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f3d4b59d5c67477bb764acda4350adb9_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:08,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/a70fc09b9c874ed29e71b863d25b3b0a, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:08,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/a70fc09b9c874ed29e71b863d25b3b0a is 175, key is test_row_0/A:col10/1732032967204/Put/seqid=0 2024-11-19T16:16:08,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742257_1433 (size=31255) 2024-11-19T16:16:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T16:16:09,296 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#A#compaction#364 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:09,296 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/cf5dd1a8bf6c4985820e67a3c71be9c5 is 175, key is test_row_0/A:col10/1732032967119/Put/seqid=0 2024-11-19T16:16:09,298 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/bc87d4ec5bcf41068bdfb8db7b05a100 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/bc87d4ec5bcf41068bdfb8db7b05a100 2024-11-19T16:16:09,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742258_1434 (size=31869) 2024-11-19T16:16:09,302 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/cf5dd1a8bf6c4985820e67a3c71be9c5 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/cf5dd1a8bf6c4985820e67a3c71be9c5 2024-11-19T16:16:09,302 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/B of 47bdefb8e31bb07f134a6eace775a473 into bc87d4ec5bcf41068bdfb8db7b05a100(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:09,302 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:09,302 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/B, priority=12, startTime=1732032968868; duration=0sec 2024-11-19T16:16:09,302 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:09,302 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:B 2024-11-19T16:16:09,303 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:09,304 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:09,304 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/C is initiating minor compaction (all files) 2024-11-19T16:16:09,304 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/C in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:09,305 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/3b378a7db3e94b39b7b8fbb32c6b217f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/d00644bbb9ca4a058e5d899e2f5d6a0a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/02970c88db274868bd4d3473923d48c8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/e918afab612f46f390b4280015e2d63b] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=48.5 K 2024-11-19T16:16:09,305 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b378a7db3e94b39b7b8fbb32c6b217f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732032963791 2024-11-19T16:16:09,305 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting d00644bbb9ca4a058e5d899e2f5d6a0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732032964411 2024-11-19T16:16:09,306 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 02970c88db274868bd4d3473923d48c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732032964549 2024-11-19T16:16:09,306 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting e918afab612f46f390b4280015e2d63b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732032965926 2024-11-19T16:16:09,306 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/A of 47bdefb8e31bb07f134a6eace775a473 into cf5dd1a8bf6c4985820e67a3c71be9c5(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:09,306 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:09,306 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/A, priority=12, startTime=1732032968867; duration=0sec 2024-11-19T16:16:09,306 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:09,306 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:A 2024-11-19T16:16:09,315 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#C#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:09,316 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/912f4c1e059d490aadecbcdc8f5cae39 is 50, key is test_row_0/C:col10/1732032967119/Put/seqid=0 2024-11-19T16:16:09,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742259_1435 (size=12915) 2024-11-19T16:16:09,351 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=439, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/a70fc09b9c874ed29e71b863d25b3b0a 2024-11-19T16:16:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/51746850fa5c4eef960bb452b4783623 is 50, key is test_row_0/B:col10/1732032967204/Put/seqid=0 2024-11-19T16:16:09,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742260_1436 (size=12301) 2024-11-19T16:16:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:09,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. as already flushing 2024-11-19T16:16:09,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033029440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:09,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033029449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:09,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033029557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:09,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033029557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:09,731 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/912f4c1e059d490aadecbcdc8f5cae39 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/912f4c1e059d490aadecbcdc8f5cae39 2024-11-19T16:16:09,734 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/C of 47bdefb8e31bb07f134a6eace775a473 into 912f4c1e059d490aadecbcdc8f5cae39(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:09,734 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:09,734 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/C, priority=12, startTime=1732032968868; duration=0sec 2024-11-19T16:16:09,734 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:09,734 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:C 2024-11-19T16:16:09,761 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/51746850fa5c4eef960bb452b4783623 2024-11-19T16:16:09,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/48b263370f354784a56fe5c52c6fcdd6 is 50, key is test_row_0/C:col10/1732032967204/Put/seqid=0 2024-11-19T16:16:09,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742261_1437 (size=12301) 2024-11-19T16:16:09,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033029777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:09,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033029777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:10,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033030087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:10,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:10,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033030087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:10,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40668 deadline: 1732033030137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:10,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40634 deadline: 1732033030137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:10,147 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 47bdefb8e31bb07f134a6eace775a473 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:10,147 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 47bdefb8e31bb07f134a6eace775a473 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T16:16:10,173 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/48b263370f354784a56fe5c52c6fcdd6 2024-11-19T16:16:10,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/a70fc09b9c874ed29e71b863d25b3b0a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a70fc09b9c874ed29e71b863d25b3b0a 2024-11-19T16:16:10,186 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a70fc09b9c874ed29e71b863d25b3b0a, entries=150, sequenceid=439, filesize=30.5 K 2024-11-19T16:16:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/51746850fa5c4eef960bb452b4783623 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/51746850fa5c4eef960bb452b4783623 2024-11-19T16:16:10,198 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/51746850fa5c4eef960bb452b4783623, entries=150, sequenceid=439, filesize=12.0 K 2024-11-19T16:16:10,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/48b263370f354784a56fe5c52c6fcdd6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/48b263370f354784a56fe5c52c6fcdd6 2024-11-19T16:16:10,203 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/48b263370f354784a56fe5c52c6fcdd6, entries=150, sequenceid=439, filesize=12.0 K 2024-11-19T16:16:10,204 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 47bdefb8e31bb07f134a6eace775a473 in 1274ms, sequenceid=439, compaction requested=false 2024-11-19T16:16:10,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:10,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:10,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-19T16:16:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-19T16:16:10,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-19T16:16:10,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1960 sec 2024-11-19T16:16:10,210 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.2010 sec 2024-11-19T16:16:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:10,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T16:16:10,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:16:10,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:10,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:16:10,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:10,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:16:10,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:10,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b90137f856fd47638a9f6172d7339bae_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032969435/Put/seqid=0 2024-11-19T16:16:10,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742262_1438 (size=17534) 2024-11-19T16:16:10,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033030681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:10,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033030683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:10,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033030787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:10,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033030796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:11,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033031007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:11,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033031016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:11,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:11,041 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b90137f856fd47638a9f6172d7339bae_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b90137f856fd47638a9f6172d7339bae_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:11,042 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/6bd4efbcab554e928e48c0d3cb7a34d3, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:11,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/6bd4efbcab554e928e48c0d3cb7a34d3 is 175, key is test_row_0/A:col10/1732032969435/Put/seqid=0 2024-11-19T16:16:11,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742263_1439 (size=48639) 2024-11-19T16:16:11,199 INFO [Thread-1631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1631. 2024-11-19T16:16:11,199 DEBUG [Thread-1631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:11,199 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:11,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:11,199 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:11,201 INFO [Thread-1635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1635. 2024-11-19T16:16:11,201 DEBUG [Thread-1635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:11,201 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:11,201 INFO [Thread-1629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1629. 2024-11-19T16:16:11,201 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:11,201 DEBUG [Thread-1629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:11,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:11,201 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:11,201 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:11,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:11,204 INFO [Thread-1633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1633. 2024-11-19T16:16:11,204 DEBUG [Thread-1633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:11,204 DEBUG [Thread-1633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:11,204 DEBUG [Thread-1633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:11,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:11,211 INFO [Thread-1637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1637. 2024-11-19T16:16:11,211 DEBUG [Thread-1637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:11,211 DEBUG [Thread-1637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:11,211 DEBUG [Thread-1637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:11,211 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:11,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033031327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:11,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033031337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:11,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:11,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:11,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:11,445 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=457, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/6bd4efbcab554e928e48c0d3cb7a34d3 2024-11-19T16:16:11,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/a619a7f63984413b82952a98ea8f27cc is 50, key is test_row_0/B:col10/1732032969435/Put/seqid=0 2024-11-19T16:16:11,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742264_1440 (size=12301) 2024-11-19T16:16:11,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40616 deadline: 1732033031836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:11,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40660 deadline: 1732033031847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:11,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:11,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:11,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:11,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/a619a7f63984413b82952a98ea8f27cc 2024-11-19T16:16:11,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/6bd8ad88dd74453993af50ad8b5c9977 is 50, key is test_row_0/C:col10/1732032969435/Put/seqid=0 2024-11-19T16:16:11,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742265_1441 (size=12301) 2024-11-19T16:16:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T16:16:12,168 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:12,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/6bd8ad88dd74453993af50ad8b5c9977 2024-11-19T16:16:12,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/6bd4efbcab554e928e48c0d3cb7a34d3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6bd4efbcab554e928e48c0d3cb7a34d3 2024-11-19T16:16:12,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6bd4efbcab554e928e48c0d3cb7a34d3, entries=250, sequenceid=457, filesize=47.5 K 2024-11-19T16:16:12,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/a619a7f63984413b82952a98ea8f27cc as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a619a7f63984413b82952a98ea8f27cc 2024-11-19T16:16:12,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a619a7f63984413b82952a98ea8f27cc, entries=150, sequenceid=457, filesize=12.0 K 2024-11-19T16:16:12,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/6bd8ad88dd74453993af50ad8b5c9977 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/6bd8ad88dd74453993af50ad8b5c9977 2024-11-19T16:16:12,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/6bd8ad88dd74453993af50ad8b5c9977, entries=150, sequenceid=457, filesize=12.0 K 2024-11-19T16:16:12,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 47bdefb8e31bb07f134a6eace775a473 in 1658ms, sequenceid=457, compaction requested=true 2024-11-19T16:16:12,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:12,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:12,283 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:12,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:12,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:12,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:12,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47bdefb8e31bb07f134a6eace775a473:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:12,283 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:12,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:12,284 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111763 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:12,284 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:12,284 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/A is initiating minor compaction (all files) 2024-11-19T16:16:12,284 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/B is initiating minor compaction (all files) 2024-11-19T16:16:12,284 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/A in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:12,284 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/B in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:12,284 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/bc87d4ec5bcf41068bdfb8db7b05a100, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/51746850fa5c4eef960bb452b4783623, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a619a7f63984413b82952a98ea8f27cc] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=36.6 K 2024-11-19T16:16:12,284 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/cf5dd1a8bf6c4985820e67a3c71be9c5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a70fc09b9c874ed29e71b863d25b3b0a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6bd4efbcab554e928e48c0d3cb7a34d3] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=109.1 K 2024-11-19T16:16:12,284 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:12,284 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/cf5dd1a8bf6c4985820e67a3c71be9c5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a70fc09b9c874ed29e71b863d25b3b0a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6bd4efbcab554e928e48c0d3cb7a34d3] 2024-11-19T16:16:12,284 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting bc87d4ec5bcf41068bdfb8db7b05a100, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732032965926 2024-11-19T16:16:12,285 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf5dd1a8bf6c4985820e67a3c71be9c5, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732032965926 2024-11-19T16:16:12,285 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 51746850fa5c4eef960bb452b4783623, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732032967188 2024-11-19T16:16:12,285 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting a70fc09b9c874ed29e71b863d25b3b0a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732032967188 2024-11-19T16:16:12,285 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a619a7f63984413b82952a98ea8f27cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1732032969435 2024-11-19T16:16:12,285 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bd4efbcab554e928e48c0d3cb7a34d3, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1732032969427 2024-11-19T16:16:12,290 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:12,291 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#B#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:12,291 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/6dd14387298548098eec4c22cf5a0f0d is 50, key is test_row_0/B:col10/1732032969435/Put/seqid=0 2024-11-19T16:16:12,292 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411195fbd4542a0ef40eea0b1991c9ce7fff3_47bdefb8e31bb07f134a6eace775a473 store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:12,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742266_1442 (size=13017) 2024-11-19T16:16:12,300 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T16:16:12,300 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411195fbd4542a0ef40eea0b1991c9ce7fff3_47bdefb8e31bb07f134a6eace775a473, store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:12,301 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195fbd4542a0ef40eea0b1991c9ce7fff3_47bdefb8e31bb07f134a6eace775a473 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:12,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742267_1443 (size=4469) 2024-11-19T16:16:12,320 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#A#compaction#373 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:12,320 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/517a96278abb44d7ba14aa22dfd1fb64 is 175, key is test_row_0/A:col10/1732032969435/Put/seqid=0 2024-11-19T16:16:12,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742268_1444 (size=31971) 2024-11-19T16:16:12,706 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/6dd14387298548098eec4c22cf5a0f0d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6dd14387298548098eec4c22cf5a0f0d 2024-11-19T16:16:12,713 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/B of 47bdefb8e31bb07f134a6eace775a473 into 6dd14387298548098eec4c22cf5a0f0d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:12,714 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:12,714 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/B, priority=13, startTime=1732032972283; duration=0sec 2024-11-19T16:16:12,714 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:12,714 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:B 2024-11-19T16:16:12,714 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:12,716 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:12,716 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 47bdefb8e31bb07f134a6eace775a473/C is initiating minor compaction (all files) 2024-11-19T16:16:12,716 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47bdefb8e31bb07f134a6eace775a473/C in TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:12,716 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/912f4c1e059d490aadecbcdc8f5cae39, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/48b263370f354784a56fe5c52c6fcdd6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/6bd8ad88dd74453993af50ad8b5c9977] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp, totalSize=36.6 K 2024-11-19T16:16:12,717 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 912f4c1e059d490aadecbcdc8f5cae39, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732032965926 2024-11-19T16:16:12,717 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 48b263370f354784a56fe5c52c6fcdd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732032967188 2024-11-19T16:16:12,718 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bd8ad88dd74453993af50ad8b5c9977, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1732032969435 2024-11-19T16:16:12,727 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47bdefb8e31bb07f134a6eace775a473#C#compaction#374 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:12,727 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/ca701d66187a41baa272f86995cadd26 is 50, key is test_row_0/C:col10/1732032969435/Put/seqid=0 2024-11-19T16:16:12,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742269_1445 (size=13017) 2024-11-19T16:16:12,733 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/517a96278abb44d7ba14aa22dfd1fb64 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/517a96278abb44d7ba14aa22dfd1fb64 2024-11-19T16:16:12,737 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/A of 47bdefb8e31bb07f134a6eace775a473 into 517a96278abb44d7ba14aa22dfd1fb64(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:12,737 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:12,737 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/A, priority=13, startTime=1732032972283; duration=0sec 2024-11-19T16:16:12,737 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:12,737 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:A 2024-11-19T16:16:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:12,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T16:16:12,848 INFO [Thread-1618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1618. 2024-11-19T16:16:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:16:12,848 DEBUG [Thread-1618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:12,848 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:16:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:16:12,848 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:12,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:12,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e0813ab67206435fa5bb3b483187b521_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_0/A:col10/1732032970682/Put/seqid=0 2024-11-19T16:16:12,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742270_1446 (size=12454) 2024-11-19T16:16:12,858 INFO [Thread-1624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1624. 2024-11-19T16:16:12,858 DEBUG [Thread-1624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:12,858 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:12,858 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:12,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:13,136 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/ca701d66187a41baa272f86995cadd26 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/ca701d66187a41baa272f86995cadd26 2024-11-19T16:16:13,140 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47bdefb8e31bb07f134a6eace775a473/C of 47bdefb8e31bb07f134a6eace775a473 into ca701d66187a41baa272f86995cadd26(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:13,140 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:13,140 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., storeName=47bdefb8e31bb07f134a6eace775a473/C, priority=13, startTime=1732032972283; duration=0sec 2024-11-19T16:16:13,140 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:13,140 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47bdefb8e31bb07f134a6eace775a473:C 2024-11-19T16:16:13,257 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:13,261 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e0813ab67206435fa5bb3b483187b521_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e0813ab67206435fa5bb3b483187b521_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:13,262 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/67ae8c522e2c4d44865eae8b0114aa3f, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:13,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/67ae8c522e2c4d44865eae8b0114aa3f is 175, key is test_row_0/A:col10/1732032970682/Put/seqid=0 2024-11-19T16:16:13,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742271_1447 (size=31255) 2024-11-19T16:16:13,681 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=477, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/67ae8c522e2c4d44865eae8b0114aa3f 2024-11-19T16:16:13,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/a4f967b1744b4d20bdc3e82b024cfb31 is 50, key is test_row_0/B:col10/1732032970682/Put/seqid=0 2024-11-19T16:16:13,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742272_1448 (size=12301) 2024-11-19T16:16:14,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/a4f967b1744b4d20bdc3e82b024cfb31 2024-11-19T16:16:14,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/c0ee7686f4c9445092de40b22e828534 is 50, key is test_row_0/C:col10/1732032970682/Put/seqid=0 2024-11-19T16:16:14,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742273_1449 (size=12301) 2024-11-19T16:16:14,181 INFO [Thread-1622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1622. 2024-11-19T16:16:14,182 DEBUG [Thread-1622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:14,182 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:14,182 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:14,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:14,189 INFO [Thread-1626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1626. 2024-11-19T16:16:14,189 DEBUG [Thread-1626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:14,189 DEBUG [Thread-1626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:14,190 DEBUG [Thread-1626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:14,190 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:14,491 INFO [Thread-1620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1620. 2024-11-19T16:16:14,491 DEBUG [Thread-1620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:14,491 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:14,492 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:14,492 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:14,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T16:16:14,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 231 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 908 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2721 rows 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 894 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2681 rows 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 900 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2699 rows 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 899 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2691 rows 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 903 2024-11-19T16:16:14,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2708 rows 2024-11-19T16:16:14,494 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T16:16:14,494 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T16:16:14,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:14,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:14,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:14,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-19T16:16:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T16:16:14,501 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032974500"}]},"ts":"1732032974500"} 2024-11-19T16:16:14,502 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T16:16:14,502 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T16:16:14,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T16:16:14,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, UNASSIGN}] 2024-11-19T16:16:14,504 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, UNASSIGN 2024-11-19T16:16:14,505 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=47bdefb8e31bb07f134a6eace775a473, regionState=CLOSING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:14,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, UNASSIGN because future has completed 2024-11-19T16:16:14,506 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T16:16:14,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:16:14,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/c0ee7686f4c9445092de40b22e828534 2024-11-19T16:16:14,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/67ae8c522e2c4d44865eae8b0114aa3f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/67ae8c522e2c4d44865eae8b0114aa3f 2024-11-19T16:16:14,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/67ae8c522e2c4d44865eae8b0114aa3f, entries=150, sequenceid=477, filesize=30.5 K 2024-11-19T16:16:14,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/a4f967b1744b4d20bdc3e82b024cfb31 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a4f967b1744b4d20bdc3e82b024cfb31 2024-11-19T16:16:14,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a4f967b1744b4d20bdc3e82b024cfb31, entries=150, sequenceid=477, filesize=12.0 K 2024-11-19T16:16:14,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/c0ee7686f4c9445092de40b22e828534 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/c0ee7686f4c9445092de40b22e828534 2024-11-19T16:16:14,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/c0ee7686f4c9445092de40b22e828534, entries=150, sequenceid=477, filesize=12.0 K 2024-11-19T16:16:14,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=26.84 KB/27480 for 47bdefb8e31bb07f134a6eace775a473 in 1682ms, sequenceid=477, compaction requested=false 2024-11-19T16:16:14,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47bdefb8e31bb07f134a6eace775a473: 2024-11-19T16:16:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T16:16:14,658 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:14,658 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T16:16:14,658 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 47bdefb8e31bb07f134a6eace775a473, disabling compactions & flushes 2024-11-19T16:16:14,658 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:14,658 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:14,658 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. after waiting 0 ms 2024-11-19T16:16:14,658 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:14,658 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 47bdefb8e31bb07f134a6eace775a473 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T16:16:14,659 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=A 2024-11-19T16:16:14,659 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:14,659 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=B 2024-11-19T16:16:14,659 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:14,659 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47bdefb8e31bb07f134a6eace775a473, store=C 2024-11-19T16:16:14,659 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:14,663 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190761ca22204a4c63bf169a1abb1d6eb8_47bdefb8e31bb07f134a6eace775a473 is 50, key is test_row_1/A:col10/1732032974488/Put/seqid=0 2024-11-19T16:16:14,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742274_1450 (size=9914) 2024-11-19T16:16:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T16:16:15,067 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:15,070 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190761ca22204a4c63bf169a1abb1d6eb8_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190761ca22204a4c63bf169a1abb1d6eb8_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:15,071 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/1907aa5b8c4241029c6603e9e361dd7d, store: [table=TestAcidGuarantees family=A region=47bdefb8e31bb07f134a6eace775a473] 2024-11-19T16:16:15,071 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/1907aa5b8c4241029c6603e9e361dd7d is 175, key is test_row_1/A:col10/1732032974488/Put/seqid=0 2024-11-19T16:16:15,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742275_1451 (size=22561) 2024-11-19T16:16:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T16:16:15,475 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=485, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/1907aa5b8c4241029c6603e9e361dd7d 2024-11-19T16:16:15,480 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/7ac3f26027354133b197dc84a8759f96 is 50, key is test_row_1/B:col10/1732032974488/Put/seqid=0 2024-11-19T16:16:15,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742276_1452 (size=9857) 2024-11-19T16:16:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T16:16:15,884 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/7ac3f26027354133b197dc84a8759f96 2024-11-19T16:16:15,890 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/bc188cc4d6b5477e94331985d709e484 is 50, key is test_row_1/C:col10/1732032974488/Put/seqid=0 2024-11-19T16:16:15,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742277_1453 (size=9857) 2024-11-19T16:16:16,293 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/bc188cc4d6b5477e94331985d709e484 2024-11-19T16:16:16,301 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/A/1907aa5b8c4241029c6603e9e361dd7d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/1907aa5b8c4241029c6603e9e361dd7d 2024-11-19T16:16:16,308 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/1907aa5b8c4241029c6603e9e361dd7d, entries=100, sequenceid=485, filesize=22.0 K 2024-11-19T16:16:16,309 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/B/7ac3f26027354133b197dc84a8759f96 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/7ac3f26027354133b197dc84a8759f96 2024-11-19T16:16:16,311 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/7ac3f26027354133b197dc84a8759f96, entries=100, sequenceid=485, filesize=9.6 K 2024-11-19T16:16:16,312 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/.tmp/C/bc188cc4d6b5477e94331985d709e484 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/bc188cc4d6b5477e94331985d709e484 2024-11-19T16:16:16,315 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/bc188cc4d6b5477e94331985d709e484, entries=100, sequenceid=485, filesize=9.6 K 2024-11-19T16:16:16,315 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 47bdefb8e31bb07f134a6eace775a473 in 1657ms, sequenceid=485, compaction requested=true 2024-11-19T16:16:16,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7fa5c5eb315f4579b793729152af08ad, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/54360e624ca547f49b86c74385d013ec, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/5f9fc138a8a14b529ef2646eacb929fd, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a7f593533d6342fcbb35a1d5960985d1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7744607ec4f945e0a843b54fe6ff823b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/bf83d25ef5ad4c8186e84f8cb238533b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3004dd40a2024a09851d3e86530db3f6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a907e34e756b4688b3b3efb95ee2671a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/023c1a4487094a09ac810d6d96d2dede, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/b948b53e5ba5404fb250f887fc9a9bf8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/0f42c473a8a242b6a99041668b9a308c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/e8a2f7b2467444a28a652b93353ffb3c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/d2ab716494cf4842b1f36d0ed3dff931, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/2370f6a4d6f54d6ca4d8b61112c572fe, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/c41bf49794d14d37b7f75ee2f0a7ebd2, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3d2b267211f245398526cece7c8d9154, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/85ac58f370ad492ba6d3a3dd806b0dd9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6cd40f7f96934859838fd11d48d74d28, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/cf5dd1a8bf6c4985820e67a3c71be9c5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a70fc09b9c874ed29e71b863d25b3b0a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6bd4efbcab554e928e48c0d3cb7a34d3] to archive 2024-11-19T16:16:16,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:16:16,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7fa5c5eb315f4579b793729152af08ad to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7fa5c5eb315f4579b793729152af08ad 2024-11-19T16:16:16,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/54360e624ca547f49b86c74385d013ec to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/54360e624ca547f49b86c74385d013ec 2024-11-19T16:16:16,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/5f9fc138a8a14b529ef2646eacb929fd to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/5f9fc138a8a14b529ef2646eacb929fd 2024-11-19T16:16:16,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a7f593533d6342fcbb35a1d5960985d1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a7f593533d6342fcbb35a1d5960985d1 2024-11-19T16:16:16,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7744607ec4f945e0a843b54fe6ff823b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/7744607ec4f945e0a843b54fe6ff823b 2024-11-19T16:16:16,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/bf83d25ef5ad4c8186e84f8cb238533b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/bf83d25ef5ad4c8186e84f8cb238533b 2024-11-19T16:16:16,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3004dd40a2024a09851d3e86530db3f6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3004dd40a2024a09851d3e86530db3f6 2024-11-19T16:16:16,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a907e34e756b4688b3b3efb95ee2671a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a907e34e756b4688b3b3efb95ee2671a 2024-11-19T16:16:16,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/023c1a4487094a09ac810d6d96d2dede to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/023c1a4487094a09ac810d6d96d2dede 2024-11-19T16:16:16,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/b948b53e5ba5404fb250f887fc9a9bf8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/b948b53e5ba5404fb250f887fc9a9bf8 2024-11-19T16:16:16,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/0f42c473a8a242b6a99041668b9a308c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/0f42c473a8a242b6a99041668b9a308c 2024-11-19T16:16:16,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/e8a2f7b2467444a28a652b93353ffb3c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/e8a2f7b2467444a28a652b93353ffb3c 2024-11-19T16:16:16,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/d2ab716494cf4842b1f36d0ed3dff931 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/d2ab716494cf4842b1f36d0ed3dff931 2024-11-19T16:16:16,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/2370f6a4d6f54d6ca4d8b61112c572fe to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/2370f6a4d6f54d6ca4d8b61112c572fe 2024-11-19T16:16:16,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/c41bf49794d14d37b7f75ee2f0a7ebd2 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/c41bf49794d14d37b7f75ee2f0a7ebd2 2024-11-19T16:16:16,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3d2b267211f245398526cece7c8d9154 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/3d2b267211f245398526cece7c8d9154 2024-11-19T16:16:16,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/85ac58f370ad492ba6d3a3dd806b0dd9 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/85ac58f370ad492ba6d3a3dd806b0dd9 2024-11-19T16:16:16,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6cd40f7f96934859838fd11d48d74d28 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6cd40f7f96934859838fd11d48d74d28 2024-11-19T16:16:16,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/cf5dd1a8bf6c4985820e67a3c71be9c5 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/cf5dd1a8bf6c4985820e67a3c71be9c5 2024-11-19T16:16:16,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a70fc09b9c874ed29e71b863d25b3b0a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/a70fc09b9c874ed29e71b863d25b3b0a 2024-11-19T16:16:16,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6bd4efbcab554e928e48c0d3cb7a34d3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/6bd4efbcab554e928e48c0d3cb7a34d3 2024-11-19T16:16:16,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/12489ca2d1554b23af7dcb934a17cf27, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/006fad54409f4f56a649854e910155a6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/f66c3b363e2c4a129c0d1acb814ed4f2, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/308a48d24c8047049f5a070c4d76ac98, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/01e3748b2a7845438923f0c796af2b78, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/78b792e4f1dc4ffda04a2ababe94a26f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/1eb2e6e92b0d49b782ee58df211789a6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/fda573e3f197453d9269c5f5f6eacbad, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6d3c740c2c4244cd906307fd3dd35270, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/9207fa66eeb04d1ab7d3df6f7e6c3479, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/4047592c80b241c389618e4152a0a7c6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/0592519292f44fbfa5263a15732b66b2, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a0505f8001e246da8c8907b32fef4248, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/0509d32c673649b891c7f41db6c4b52f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6f18595d1a2d41d7bfbe1b41b1cb429b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/62a5655eed264913b5183a59f1b6fc4b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/080f8e8ad46849b89971e4eddf8f6479, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/bc87d4ec5bcf41068bdfb8db7b05a100, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/b3726a0a8a8b4399b56a7dae6bc9cf01, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/51746850fa5c4eef960bb452b4783623, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a619a7f63984413b82952a98ea8f27cc] to archive 2024-11-19T16:16:16,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:16:16,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/12489ca2d1554b23af7dcb934a17cf27 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/12489ca2d1554b23af7dcb934a17cf27 2024-11-19T16:16:16,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/006fad54409f4f56a649854e910155a6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/006fad54409f4f56a649854e910155a6 2024-11-19T16:16:16,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/f66c3b363e2c4a129c0d1acb814ed4f2 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/f66c3b363e2c4a129c0d1acb814ed4f2 2024-11-19T16:16:16,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/308a48d24c8047049f5a070c4d76ac98 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/308a48d24c8047049f5a070c4d76ac98 2024-11-19T16:16:16,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/01e3748b2a7845438923f0c796af2b78 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/01e3748b2a7845438923f0c796af2b78 2024-11-19T16:16:16,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/78b792e4f1dc4ffda04a2ababe94a26f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/78b792e4f1dc4ffda04a2ababe94a26f 2024-11-19T16:16:16,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/1eb2e6e92b0d49b782ee58df211789a6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/1eb2e6e92b0d49b782ee58df211789a6 2024-11-19T16:16:16,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/fda573e3f197453d9269c5f5f6eacbad to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/fda573e3f197453d9269c5f5f6eacbad 2024-11-19T16:16:16,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6d3c740c2c4244cd906307fd3dd35270 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6d3c740c2c4244cd906307fd3dd35270 2024-11-19T16:16:16,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/9207fa66eeb04d1ab7d3df6f7e6c3479 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/9207fa66eeb04d1ab7d3df6f7e6c3479 2024-11-19T16:16:16,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/4047592c80b241c389618e4152a0a7c6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/4047592c80b241c389618e4152a0a7c6 2024-11-19T16:16:16,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/0592519292f44fbfa5263a15732b66b2 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/0592519292f44fbfa5263a15732b66b2 2024-11-19T16:16:16,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a0505f8001e246da8c8907b32fef4248 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a0505f8001e246da8c8907b32fef4248 2024-11-19T16:16:16,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/0509d32c673649b891c7f41db6c4b52f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/0509d32c673649b891c7f41db6c4b52f 2024-11-19T16:16:16,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6f18595d1a2d41d7bfbe1b41b1cb429b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6f18595d1a2d41d7bfbe1b41b1cb429b 2024-11-19T16:16:16,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/62a5655eed264913b5183a59f1b6fc4b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/62a5655eed264913b5183a59f1b6fc4b 2024-11-19T16:16:16,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/080f8e8ad46849b89971e4eddf8f6479 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/080f8e8ad46849b89971e4eddf8f6479 2024-11-19T16:16:16,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/bc87d4ec5bcf41068bdfb8db7b05a100 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/bc87d4ec5bcf41068bdfb8db7b05a100 2024-11-19T16:16:16,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/b3726a0a8a8b4399b56a7dae6bc9cf01 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/b3726a0a8a8b4399b56a7dae6bc9cf01 2024-11-19T16:16:16,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/51746850fa5c4eef960bb452b4783623 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/51746850fa5c4eef960bb452b4783623 2024-11-19T16:16:16,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a619a7f63984413b82952a98ea8f27cc to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a619a7f63984413b82952a98ea8f27cc 2024-11-19T16:16:16,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/5b895b4fd2734696b98d567afbe7cefd, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/7650241d84be4749b1fef22638a49655, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/e9099603d6b4422b87faa5cea1695d17, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/77c78d0f06a14a74979c8226eb2dfa3d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/8665ec1916a04135bce42c076f0e8e6c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/d4d3e30eb1b94071ad913d54670b0a3e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/a3aee8ff491041179d45666919ce08bc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/4c24ca1356ba4820987f12d3edefd9ad, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/866e2a66279a492bbc0518a09165477e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/c99038426115469a8581470e13af25a9, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/0ce23a1883f647f1ac91a456bb0c6183, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/37ec1ac5537746eaa9da0f37dba60451, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/cd7f516f5ee24afaa14b9d06e7031a46, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/3b378a7db3e94b39b7b8fbb32c6b217f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/a52bde9f23d64737bc968c17d75a0062, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/d00644bbb9ca4a058e5d899e2f5d6a0a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/02970c88db274868bd4d3473923d48c8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/912f4c1e059d490aadecbcdc8f5cae39, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/e918afab612f46f390b4280015e2d63b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/48b263370f354784a56fe5c52c6fcdd6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/6bd8ad88dd74453993af50ad8b5c9977] to archive 2024-11-19T16:16:16,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:16:16,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/5b895b4fd2734696b98d567afbe7cefd to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/5b895b4fd2734696b98d567afbe7cefd 2024-11-19T16:16:16,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/7650241d84be4749b1fef22638a49655 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/7650241d84be4749b1fef22638a49655 2024-11-19T16:16:16,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/e9099603d6b4422b87faa5cea1695d17 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/e9099603d6b4422b87faa5cea1695d17 2024-11-19T16:16:16,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/77c78d0f06a14a74979c8226eb2dfa3d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/77c78d0f06a14a74979c8226eb2dfa3d 2024-11-19T16:16:16,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/8665ec1916a04135bce42c076f0e8e6c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/8665ec1916a04135bce42c076f0e8e6c 2024-11-19T16:16:16,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/d4d3e30eb1b94071ad913d54670b0a3e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/d4d3e30eb1b94071ad913d54670b0a3e 2024-11-19T16:16:16,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/a3aee8ff491041179d45666919ce08bc to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/a3aee8ff491041179d45666919ce08bc 2024-11-19T16:16:16,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/4c24ca1356ba4820987f12d3edefd9ad to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/4c24ca1356ba4820987f12d3edefd9ad 2024-11-19T16:16:16,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/866e2a66279a492bbc0518a09165477e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/866e2a66279a492bbc0518a09165477e 2024-11-19T16:16:16,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/c99038426115469a8581470e13af25a9 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/c99038426115469a8581470e13af25a9 2024-11-19T16:16:16,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/0ce23a1883f647f1ac91a456bb0c6183 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/0ce23a1883f647f1ac91a456bb0c6183 2024-11-19T16:16:16,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/37ec1ac5537746eaa9da0f37dba60451 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/37ec1ac5537746eaa9da0f37dba60451 2024-11-19T16:16:16,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/cd7f516f5ee24afaa14b9d06e7031a46 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/cd7f516f5ee24afaa14b9d06e7031a46 2024-11-19T16:16:16,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/3b378a7db3e94b39b7b8fbb32c6b217f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/3b378a7db3e94b39b7b8fbb32c6b217f 2024-11-19T16:16:16,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/a52bde9f23d64737bc968c17d75a0062 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/a52bde9f23d64737bc968c17d75a0062 2024-11-19T16:16:16,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/d00644bbb9ca4a058e5d899e2f5d6a0a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/d00644bbb9ca4a058e5d899e2f5d6a0a 2024-11-19T16:16:16,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/02970c88db274868bd4d3473923d48c8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/02970c88db274868bd4d3473923d48c8 2024-11-19T16:16:16,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/912f4c1e059d490aadecbcdc8f5cae39 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/912f4c1e059d490aadecbcdc8f5cae39 2024-11-19T16:16:16,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/e918afab612f46f390b4280015e2d63b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/e918afab612f46f390b4280015e2d63b 2024-11-19T16:16:16,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/48b263370f354784a56fe5c52c6fcdd6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/48b263370f354784a56fe5c52c6fcdd6 2024-11-19T16:16:16,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/6bd8ad88dd74453993af50ad8b5c9977 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/6bd8ad88dd74453993af50ad8b5c9977 2024-11-19T16:16:16,368 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/recovered.edits/488.seqid, newMaxSeqId=488, maxSeqId=4 2024-11-19T16:16:16,368 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. 2024-11-19T16:16:16,368 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 47bdefb8e31bb07f134a6eace775a473: Waiting for close lock at 1732032974658Running coprocessor pre-close hooks at 1732032974658Disabling compacts and flushes for region at 1732032974658Disabling writes for close at 1732032974658Obtaining lock to block concurrent updates at 1732032974658Preparing flush snapshotting stores in 47bdefb8e31bb07f134a6eace775a473 at 1732032974658Finished memstore snapshotting TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732032974659 (+1 ms)Flushing stores of TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473. at 1732032974659Flushing 47bdefb8e31bb07f134a6eace775a473/A: creating writer at 1732032974659Flushing mob file 47bdefb8e31bb07f134a6eace775a473/A: appending metadata at 1732032974663 (+4 ms)Flushing mob file 47bdefb8e31bb07f134a6eace775a473/A: closing flushed file at 1732032974663Flushing 47bdefb8e31bb07f134a6eace775a473/A: appending metadata at 1732032975071 (+408 ms)Flushing 47bdefb8e31bb07f134a6eace775a473/A: closing flushed file at 1732032975071Flushing 47bdefb8e31bb07f134a6eace775a473/B: creating writer at 1732032975478 (+407 ms)Flushing 47bdefb8e31bb07f134a6eace775a473/B: appending metadata at 1732032975480 (+2 ms)Flushing 47bdefb8e31bb07f134a6eace775a473/B: closing flushed file at 1732032975480Flushing 47bdefb8e31bb07f134a6eace775a473/C: creating writer at 1732032975887 (+407 ms)Flushing 47bdefb8e31bb07f134a6eace775a473/C: appending metadata at 1732032975889 (+2 ms)Flushing 47bdefb8e31bb07f134a6eace775a473/C: closing flushed file at 1732032975889Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51b5e098: reopening flushed file at 1732032976300 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40993dfd: reopening flushed file at 1732032976308 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@143b3308: reopening flushed file at 1732032976311 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 47bdefb8e31bb07f134a6eace775a473 in 1657ms, sequenceid=485, compaction requested=true at 1732032976315 (+4 ms)Writing region close event to WAL at 1732032976366 (+51 ms)Running coprocessor post-close hooks at 1732032976368 (+2 ms)Closed at 1732032976368 2024-11-19T16:16:16,370 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,370 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=47bdefb8e31bb07f134a6eace775a473, regionState=CLOSED 2024-11-19T16:16:16,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:16:16,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-19T16:16:16,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 47bdefb8e31bb07f134a6eace775a473, server=32a8860bb9d6,43095,1732032854050 in 1.8660 sec 2024-11-19T16:16:16,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-19T16:16:16,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47bdefb8e31bb07f134a6eace775a473, UNASSIGN in 1.8690 sec 2024-11-19T16:16:16,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-19T16:16:16,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8730 sec 2024-11-19T16:16:16,376 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032976376"}]},"ts":"1732032976376"} 2024-11-19T16:16:16,377 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T16:16:16,377 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T16:16:16,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8820 sec 2024-11-19T16:16:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T16:16:16,648 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:16,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-19T16:16:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:16,650 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T16:16:16,651 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:16,653 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,655 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/recovered.edits] 2024-11-19T16:16:16,657 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/1907aa5b8c4241029c6603e9e361dd7d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/1907aa5b8c4241029c6603e9e361dd7d 2024-11-19T16:16:16,658 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/517a96278abb44d7ba14aa22dfd1fb64 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/517a96278abb44d7ba14aa22dfd1fb64 2024-11-19T16:16:16,659 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/67ae8c522e2c4d44865eae8b0114aa3f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/A/67ae8c522e2c4d44865eae8b0114aa3f 2024-11-19T16:16:16,661 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6dd14387298548098eec4c22cf5a0f0d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/6dd14387298548098eec4c22cf5a0f0d 2024-11-19T16:16:16,662 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/7ac3f26027354133b197dc84a8759f96 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/7ac3f26027354133b197dc84a8759f96 2024-11-19T16:16:16,663 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a4f967b1744b4d20bdc3e82b024cfb31 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/B/a4f967b1744b4d20bdc3e82b024cfb31 2024-11-19T16:16:16,665 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/bc188cc4d6b5477e94331985d709e484 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/bc188cc4d6b5477e94331985d709e484 2024-11-19T16:16:16,667 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/c0ee7686f4c9445092de40b22e828534 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/c0ee7686f4c9445092de40b22e828534 2024-11-19T16:16:16,668 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/ca701d66187a41baa272f86995cadd26 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/C/ca701d66187a41baa272f86995cadd26 2024-11-19T16:16:16,671 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/recovered.edits/488.seqid to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473/recovered.edits/488.seqid 2024-11-19T16:16:16,672 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,672 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T16:16:16,672 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T16:16:16,673 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T16:16:16,675 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119075d79a89ffa4ac98ad83ca49e660c90_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119075d79a89ffa4ac98ad83ca49e660c90_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,676 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190761ca22204a4c63bf169a1abb1d6eb8_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190761ca22204a4c63bf169a1abb1d6eb8_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,677 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190e4462f03b3d4ba99070c17d7543f480_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190e4462f03b3d4ba99070c17d7543f480_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,678 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119237256b42cec412b85fc67252dac4088_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119237256b42cec412b85fc67252dac4088_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,679 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192540427fa4f940f6910abe4fe64f228a_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192540427fa4f940f6910abe4fe64f228a_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,680 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192cb160522142456aa5e6032a84e74a55_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192cb160522142456aa5e6032a84e74a55_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,681 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196e32cebf5041466382fe0a713dcb56d9_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196e32cebf5041466382fe0a713dcb56d9_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,681 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119865193e9a43a4c56a7c00eef3846f999_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119865193e9a43a4c56a7c00eef3846f999_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,682 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111988b5e57a5e0a48faa41711d9c7841f7d_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111988b5e57a5e0a48faa41711d9c7841f7d_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,683 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198b8691c9272d4dab9be9490533eb0f2e_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198b8691c9272d4dab9be9490533eb0f2e_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,684 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b086ee0b6689482c88142d6e634a9d32_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b086ee0b6689482c88142d6e634a9d32_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,685 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b90137f856fd47638a9f6172d7339bae_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b90137f856fd47638a9f6172d7339bae_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,685 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c6f1e26a33404f0c95fd253f1f04af8c_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c6f1e26a33404f0c95fd253f1f04af8c_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,686 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c7d04ddb4226407e91739f281631ccc6_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c7d04ddb4226407e91739f281631ccc6_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,687 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d4cc3820c716491fbbe7ecf65d64ae35_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d4cc3820c716491fbbe7ecf65d64ae35_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,688 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e0813ab67206435fa5bb3b483187b521_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e0813ab67206435fa5bb3b483187b521_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,689 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ed8f2b7265424b77bcca36b755de9e70_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ed8f2b7265424b77bcca36b755de9e70_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,690 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f3d4b59d5c67477bb764acda4350adb9_47bdefb8e31bb07f134a6eace775a473 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f3d4b59d5c67477bb764acda4350adb9_47bdefb8e31bb07f134a6eace775a473 2024-11-19T16:16:16,690 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T16:16:16,692 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:16,694 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T16:16:16,696 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T16:16:16,697 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:16,697 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T16:16:16,697 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732032976697"}]},"ts":"9223372036854775807"} 2024-11-19T16:16:16,698 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T16:16:16,698 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 47bdefb8e31bb07f134a6eace775a473, NAME => 'TestAcidGuarantees,,1732032949064.47bdefb8e31bb07f134a6eace775a473.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T16:16:16,698 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T16:16:16,698 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732032976698"}]},"ts":"9223372036854775807"} 2024-11-19T16:16:16,699 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T16:16:16,700 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:16,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-11-19T16:16:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T16:16:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T16:16:16,759 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:16,775 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-607038221_22 at /127.0.0.1:44262 [Waiting for operation #1229] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2117921560_22 at /127.0.0.1:41498 [Waiting for operation #1173] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-607038221_22 at /127.0.0.1:41518 [Waiting for operation #1156] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2117921560_22 at /127.0.0.1:41510 [Waiting for operation #1202] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 424) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4057 (was 4574) 2024-11-19T16:16:16,784 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=4057 2024-11-19T16:16:16,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T16:16:16,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T16:16:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:16,787 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T16:16:16,787 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:16,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-19T16:16:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-19T16:16:16,788 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T16:16:16,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742278_1454 (size=1061) 2024-11-19T16:16:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-19T16:16:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-19T16:16:17,194 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70 2024-11-19T16:16:17,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742279_1455 (size=53) 2024-11-19T16:16:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-19T16:16:17,505 WARN [regionserver/32a8860bb9d6:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-11-19T16:16:17,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:16:17,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 38dce676c0e0bc7474a9f822adbf177b, disabling compactions & flushes 2024-11-19T16:16:17,599 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:17,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:17,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. after waiting 0 ms 2024-11-19T16:16:17,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:17,599 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:17,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 38dce676c0e0bc7474a9f822adbf177b: Waiting for close lock at 1732032977599Disabling compacts and flushes for region at 1732032977599Disabling writes for close at 1732032977599Writing region close event to WAL at 1732032977599Closed at 1732032977599 2024-11-19T16:16:17,600 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T16:16:17,600 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732032977600"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732032977600"}]},"ts":"1732032977600"} 2024-11-19T16:16:17,602 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T16:16:17,602 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T16:16:17,602 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032977602"}]},"ts":"1732032977602"} 2024-11-19T16:16:17,603 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T16:16:17,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38dce676c0e0bc7474a9f822adbf177b, ASSIGN}] 2024-11-19T16:16:17,604 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38dce676c0e0bc7474a9f822adbf177b, ASSIGN 2024-11-19T16:16:17,605 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38dce676c0e0bc7474a9f822adbf177b, ASSIGN; state=OFFLINE, location=32a8860bb9d6,43095,1732032854050; forceNewPlan=false, retain=false 2024-11-19T16:16:17,756 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=38dce676c0e0bc7474a9f822adbf177b, regionState=OPENING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:17,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38dce676c0e0bc7474a9f822adbf177b, ASSIGN because future has completed 2024-11-19T16:16:17,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:16:17,913 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:17,913 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} 2024-11-19T16:16:17,914 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:17,914 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:16:17,915 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:17,915 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:17,917 INFO [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:17,919 INFO [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:16:17,919 INFO [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38dce676c0e0bc7474a9f822adbf177b columnFamilyName A 2024-11-19T16:16:17,919 DEBUG [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:17,920 INFO [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] regionserver.HStore(327): Store=38dce676c0e0bc7474a9f822adbf177b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:16:17,921 INFO [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:17,923 INFO [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:16:17,923 INFO [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38dce676c0e0bc7474a9f822adbf177b columnFamilyName B 2024-11-19T16:16:17,923 DEBUG [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:17,924 INFO [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] regionserver.HStore(327): Store=38dce676c0e0bc7474a9f822adbf177b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:16:17,924 INFO [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:17,925 INFO [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:16:17,925 INFO [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38dce676c0e0bc7474a9f822adbf177b columnFamilyName C 2024-11-19T16:16:17,926 DEBUG [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:17,926 INFO [StoreOpener-38dce676c0e0bc7474a9f822adbf177b-1 {}] regionserver.HStore(327): Store=38dce676c0e0bc7474a9f822adbf177b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:16:17,926 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:17,926 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:17,927 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-19T16:16:17,928 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:17,929 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:17,929 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:17,930 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T16:16:17,931 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:17,934 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T16:16:17,934 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 38dce676c0e0bc7474a9f822adbf177b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60980432, jitterRate=-0.09132075309753418}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T16:16:17,935 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:17,936 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 38dce676c0e0bc7474a9f822adbf177b: Running coprocessor pre-open hook at 1732032977915Writing region info on filesystem at 1732032977915Initializing all the Stores at 1732032977916 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032977916Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032977917 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732032977917Cleaning up temporary data from old regions at 1732032977929 (+12 ms)Running coprocessor post-open hooks at 1732032977935 (+6 ms)Region opened successfully at 1732032977936 (+1 ms) 2024-11-19T16:16:17,937 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., pid=114, masterSystemTime=1732032977909 2024-11-19T16:16:17,940 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:17,940 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:17,940 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=38dce676c0e0bc7474a9f822adbf177b, regionState=OPEN, openSeqNum=2, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:17,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:16:17,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-19T16:16:17,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 in 186 msec 2024-11-19T16:16:17,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-19T16:16:17,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38dce676c0e0bc7474a9f822adbf177b, ASSIGN in 341 msec 2024-11-19T16:16:17,948 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T16:16:17,948 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732032977948"}]},"ts":"1732032977948"} 2024-11-19T16:16:17,949 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T16:16:17,950 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T16:16:17,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-11-19T16:16:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-19T16:16:18,947 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:18,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:18,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:18,950 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd5dbb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:18,953 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c0b428c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,954 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:18,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:18,958 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T16:16:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 32a8860bb9d6,37355,1732032853237 2024-11-19T16:16:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5344c64b 2024-11-19T16:16:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T16:16:18,961 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T16:16:18,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:18,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:18,964 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69176434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:18,965 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b198696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:18,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:18,967 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67790312, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:18,968 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c12dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:18,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:18,969 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51cad91a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:18,970 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:18,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d707e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:18,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:18,972 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e3373ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:18,973 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9bdc2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:18,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:18,975 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e7bf54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:18,976 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e4a878e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:18,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:18,977 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72a09504, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:18,979 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5aa0cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:18,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:18,980 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@678a5d88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:18,981 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54cf0181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:18,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:18,983 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37feb0b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:18,984 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747a0fa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:18,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:18,986 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f791ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:18,987 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3261b08c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:18,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:18,989 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1285ce3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:18,990 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363612a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:18,991 DEBUG [Thread-2043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:18,991 DEBUG [Thread-2041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:18,991 DEBUG [Thread-2058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:18,991 DEBUG [Thread-2050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:18,991 DEBUG [Thread-2056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:18,991 DEBUG [Thread-2047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:18,992 DEBUG [Thread-2039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:18,992 DEBUG [Thread-2041 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:18,993 DEBUG [Thread-2041 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:18,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:18,993 DEBUG [Thread-2045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:18,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:18,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:18,994 DEBUG [Thread-2052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:18,994 DEBUG [Thread-2054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:18,995 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:16:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:18,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:18,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:18,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:18,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:16:18,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:16:18,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:16:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:18,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-19T16:16:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:16:19,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T16:16:19,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:19,001 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:19,002 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:16:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:16:19,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:19,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2] 2024-11-19T16:16:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:19,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:16:19,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:19,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:19,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:19,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:19,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:19,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:19,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/d3c5bdf78399428a8ff1e85fd16b5f98 is 50, key is test_row_0/A:col10/1732032979009/Put/seqid=0 2024-11-19T16:16:19,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742280_1456 (size=11997) 2024-11-19T16:16:19,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732032979021, max timestamp=1732032979057Num uniques 859; 2024-11-19T16:16:19,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732032979021, max timestamp=1732032979057Num uniques 859; 2024-11-19T16:16:19,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:16:19,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T16:16:19,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732032979021, max timestamp=1732032979057Num uniques 859; 2024-11-19T16:16:19,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T16:16:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T16:16:19,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44532 deadline: 1732033039119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033039120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033039121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033039122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033039132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T16:16:19,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:19,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:19,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:19,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44532 deadline: 1732033039236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033039236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033039236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033039236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033039247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T16:16:19,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:19,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:19,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:19,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T16:16:19,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/d3c5bdf78399428a8ff1e85fd16b5f98 2024-11-19T16:16:19,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44532 deadline: 1732033039447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033039447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033039447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033039447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T16:16:19,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:19,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:19,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:19,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033039467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/bc78110ff7734933a900a8c6e72ce6b1 is 50, key is test_row_0/B:col10/1732032979009/Put/seqid=0 2024-11-19T16:16:19,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742281_1457 (size=9657) 2024-11-19T16:16:19,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T16:16:19,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:19,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:19,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:19,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T16:16:19,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44532 deadline: 1732033039757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033039757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T16:16:19,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:19,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033039767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033039767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033039777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:19,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/bc78110ff7734933a900a8c6e72ce6b1 2024-11-19T16:16:19,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/bbb0e651b0f144b5bbaa667dc6019d71 is 50, key is test_row_0/C:col10/1732032979009/Put/seqid=0 2024-11-19T16:16:19,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742282_1458 (size=9657) 2024-11-19T16:16:19,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T16:16:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:19,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:20,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T16:16:20,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:20,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:20,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:20,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:20,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T16:16:20,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T16:16:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:20,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:20,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44532 deadline: 1732033040277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:20,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033040277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:20,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033040287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:20,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033040287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:20,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:20,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:20,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:20,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/bbb0e651b0f144b5bbaa667dc6019d71 2024-11-19T16:16:20,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033040296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:20,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/d3c5bdf78399428a8ff1e85fd16b5f98 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/d3c5bdf78399428a8ff1e85fd16b5f98 2024-11-19T16:16:20,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/d3c5bdf78399428a8ff1e85fd16b5f98, entries=150, sequenceid=11, filesize=11.7 K 2024-11-19T16:16:20,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/bc78110ff7734933a900a8c6e72ce6b1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/bc78110ff7734933a900a8c6e72ce6b1 2024-11-19T16:16:20,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/bc78110ff7734933a900a8c6e72ce6b1, entries=100, sequenceid=11, filesize=9.4 K 2024-11-19T16:16:20,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/bbb0e651b0f144b5bbaa667dc6019d71 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/bbb0e651b0f144b5bbaa667dc6019d71 2024-11-19T16:16:20,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/bbb0e651b0f144b5bbaa667dc6019d71, entries=100, sequenceid=11, filesize=9.4 K 2024-11-19T16:16:20,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.02 KB/139286 for 38dce676c0e0bc7474a9f822adbf177b in 1305ms, sequenceid=11, compaction requested=false 2024-11-19T16:16:20,316 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T16:16:20,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:20,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T16:16:20,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:20,415 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=136.02 KB heapSize=384.46 KB 2024-11-19T16:16:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:20,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/d0c60533027c4bcdb621695662a278af is 50, key is test_row_0/A:col10/1732032979118/Put/seqid=0 2024-11-19T16:16:20,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742283_1459 (size=12001) 2024-11-19T16:16:20,823 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/d0c60533027c4bcdb621695662a278af 2024-11-19T16:16:20,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/f2efb843bea845e489fd634f4886e972 is 50, key is test_row_0/B:col10/1732032979118/Put/seqid=0 2024-11-19T16:16:20,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742284_1460 (size=12001) 2024-11-19T16:16:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T16:16:21,233 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/f2efb843bea845e489fd634f4886e972 2024-11-19T16:16:21,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/6b84ba8f4feb417ca25d56a8977b9853 is 50, key is test_row_0/C:col10/1732032979118/Put/seqid=0 2024-11-19T16:16:21,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742285_1461 (size=12001) 2024-11-19T16:16:21,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:21,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44532 deadline: 1732033041314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033041315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033041316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033041316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033041317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,332 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 38dce676c0e0bc7474a9f822adbf177b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:21,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44532 deadline: 1732033041426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033041437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033041437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033041437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,648 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.10 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/6b84ba8f4feb417ca25d56a8977b9853 2024-11-19T16:16:21,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/d0c60533027c4bcdb621695662a278af as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/d0c60533027c4bcdb621695662a278af 2024-11-19T16:16:21,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44532 deadline: 1732033041646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,656 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/d0c60533027c4bcdb621695662a278af, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T16:16:21,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/f2efb843bea845e489fd634f4886e972 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/f2efb843bea845e489fd634f4886e972 2024-11-19T16:16:21,659 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/f2efb843bea845e489fd634f4886e972, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T16:16:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/6b84ba8f4feb417ca25d56a8977b9853 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/6b84ba8f4feb417ca25d56a8977b9853 2024-11-19T16:16:21,662 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/6b84ba8f4feb417ca25d56a8977b9853, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T16:16:21,662 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~136.02 KB/139286, heapSize ~384.41 KB/393640, currentSize=46.96 KB/48090 for 38dce676c0e0bc7474a9f822adbf177b in 1247ms, sequenceid=47, compaction requested=false 2024-11-19T16:16:21,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:21,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:21,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-19T16:16:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-19T16:16:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:21,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T16:16:21,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:21,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:21,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:21,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:21,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:21,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:21,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-19T16:16:21,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6630 sec 2024-11-19T16:16:21,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.6720 sec 2024-11-19T16:16:21,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/81a329ccb88b4759bb3fd30aee401296 is 50, key is test_row_0/A:col10/1732032981316/Put/seqid=0 2024-11-19T16:16:21,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742286_1462 (size=14341) 2024-11-19T16:16:21,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032981699, max timestamp=1732032981719Num uniques 859; 2024-11-19T16:16:21,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032981699, max timestamp=1732032981719Num uniques 859; 2024-11-19T16:16:21,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:21,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:21,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032981699, max timestamp=1732032981719Num uniques 859; 2024-11-19T16:16:21,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:21,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033041762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033041763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033041774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033041877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033041877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033041887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:21,890 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T16:16:21,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44532 deadline: 1732033041966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/81a329ccb88b4759bb3fd30aee401296 2024-11-19T16:16:22,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/b9a9585f47f04fadb8f50c6b5c17d45f is 50, key is test_row_0/B:col10/1732032981316/Put/seqid=0 2024-11-19T16:16:22,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742287_1463 (size=12001) 2024-11-19T16:16:22,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/b9a9585f47f04fadb8f50c6b5c17d45f 2024-11-19T16:16:22,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/d0fe68cbfbaa4602be716fb0592a61d8 is 50, key is test_row_0/C:col10/1732032981316/Put/seqid=0 2024-11-19T16:16:22,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742288_1464 (size=12001) 2024-11-19T16:16:22,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/d0fe68cbfbaa4602be716fb0592a61d8 2024-11-19T16:16:22,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033042097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:22,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033042097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:22,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033042097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/81a329ccb88b4759bb3fd30aee401296 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/81a329ccb88b4759bb3fd30aee401296 2024-11-19T16:16:22,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/81a329ccb88b4759bb3fd30aee401296, entries=200, sequenceid=58, filesize=14.0 K 2024-11-19T16:16:22,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/b9a9585f47f04fadb8f50c6b5c17d45f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/b9a9585f47f04fadb8f50c6b5c17d45f 2024-11-19T16:16:22,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/b9a9585f47f04fadb8f50c6b5c17d45f, entries=150, sequenceid=58, filesize=11.7 K 2024-11-19T16:16:22,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/d0fe68cbfbaa4602be716fb0592a61d8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/d0fe68cbfbaa4602be716fb0592a61d8 2024-11-19T16:16:22,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/d0fe68cbfbaa4602be716fb0592a61d8, entries=150, sequenceid=58, filesize=11.7 K 2024-11-19T16:16:22,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 38dce676c0e0bc7474a9f822adbf177b in 472ms, sequenceid=58, compaction requested=true 2024-11-19T16:16:22,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:22,138 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:22,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:22,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:22,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:22,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:22,139 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:22,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:22,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:22,139 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:22,139 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/A is initiating minor compaction (all files) 2024-11-19T16:16:22,139 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/A in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:22,139 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/d3c5bdf78399428a8ff1e85fd16b5f98, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/d0c60533027c4bcdb621695662a278af, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/81a329ccb88b4759bb3fd30aee401296] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=37.4 K 2024-11-19T16:16:22,139 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:22,140 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/B is initiating minor compaction (all files) 2024-11-19T16:16:22,140 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/B in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:22,140 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3c5bdf78399428a8ff1e85fd16b5f98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732032979000 2024-11-19T16:16:22,140 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/bc78110ff7734933a900a8c6e72ce6b1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/f2efb843bea845e489fd634f4886e972, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/b9a9585f47f04fadb8f50c6b5c17d45f] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=32.9 K 2024-11-19T16:16:22,141 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0c60533027c4bcdb621695662a278af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732032979117 2024-11-19T16:16:22,141 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting bc78110ff7734933a900a8c6e72ce6b1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732032979000 2024-11-19T16:16:22,141 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81a329ccb88b4759bb3fd30aee401296, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032981306 2024-11-19T16:16:22,141 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting f2efb843bea845e489fd634f4886e972, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732032979117 2024-11-19T16:16:22,142 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a9585f47f04fadb8f50c6b5c17d45f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032981306 2024-11-19T16:16:22,151 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#B#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:22,152 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/4138b58532f74ae1847d5d0e525e1afc is 50, key is test_row_0/B:col10/1732032981316/Put/seqid=0 2024-11-19T16:16:22,158 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#A#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:22,159 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/220bbb749bdf499bb0c35d69d4db59ec is 50, key is test_row_0/A:col10/1732032981316/Put/seqid=0 2024-11-19T16:16:22,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742289_1465 (size=12104) 2024-11-19T16:16:22,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742290_1466 (size=12104) 2024-11-19T16:16:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:22,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T16:16:22,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:22,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:22,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:22,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:22,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:22,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:22,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/f610728d5a364301a27e414995c7890c is 50, key is test_row_0/A:col10/1732032981759/Put/seqid=0 2024-11-19T16:16:22,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742291_1467 (size=14341) 2024-11-19T16:16:22,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033042448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:22,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033042452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:22,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033042453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44532 deadline: 1732033042487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:22,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033042557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033042577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:22,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033042577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,589 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/4138b58532f74ae1847d5d0e525e1afc as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4138b58532f74ae1847d5d0e525e1afc 2024-11-19T16:16:22,593 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/B of 38dce676c0e0bc7474a9f822adbf177b into 4138b58532f74ae1847d5d0e525e1afc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:22,593 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:22,593 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/B, priority=13, startTime=1732032982138; duration=0sec 2024-11-19T16:16:22,594 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:22,594 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:B 2024-11-19T16:16:22,594 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:22,594 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:22,595 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/C is initiating minor compaction (all files) 2024-11-19T16:16:22,595 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/C in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:22,595 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/bbb0e651b0f144b5bbaa667dc6019d71, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/6b84ba8f4feb417ca25d56a8977b9853, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/d0fe68cbfbaa4602be716fb0592a61d8] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=32.9 K 2024-11-19T16:16:22,595 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting bbb0e651b0f144b5bbaa667dc6019d71, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732032979000 2024-11-19T16:16:22,595 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b84ba8f4feb417ca25d56a8977b9853, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732032979117 2024-11-19T16:16:22,595 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting d0fe68cbfbaa4602be716fb0592a61d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032981306 2024-11-19T16:16:22,598 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/220bbb749bdf499bb0c35d69d4db59ec as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/220bbb749bdf499bb0c35d69d4db59ec 2024-11-19T16:16:22,601 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#C#compaction#393 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:22,601 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/66c7127e019b4de08d0267842dd3cf50 is 50, key is test_row_0/C:col10/1732032981316/Put/seqid=0 2024-11-19T16:16:22,601 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/A of 38dce676c0e0bc7474a9f822adbf177b into 220bbb749bdf499bb0c35d69d4db59ec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:22,601 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:22,601 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/A, priority=13, startTime=1732032982138; duration=0sec 2024-11-19T16:16:22,601 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:22,601 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:A 2024-11-19T16:16:22,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742292_1468 (size=12104) 2024-11-19T16:16:22,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033042776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:22,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033042797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:22,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033042797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:22,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/f610728d5a364301a27e414995c7890c 2024-11-19T16:16:22,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/7cd7283370524e9395397256d0cbc01b is 50, key is test_row_0/B:col10/1732032981759/Put/seqid=0 2024-11-19T16:16:22,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742293_1469 (size=12001) 2024-11-19T16:16:23,007 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/66c7127e019b4de08d0267842dd3cf50 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/66c7127e019b4de08d0267842dd3cf50 2024-11-19T16:16:23,010 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/C of 38dce676c0e0bc7474a9f822adbf177b into 66c7127e019b4de08d0267842dd3cf50(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:23,010 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:23,010 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/C, priority=13, startTime=1732032982139; duration=0sec 2024-11-19T16:16:23,010 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:23,010 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:C 2024-11-19T16:16:23,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033043087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:23,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033043116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:23,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:23,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033043116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:23,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:23,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T16:16:23,167 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:23,168 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-19T16:16:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T16:16:23,169 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:23,170 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:23,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:23,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/7cd7283370524e9395397256d0cbc01b 2024-11-19T16:16:23,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/b317995e536044e0a9b01b053f623962 is 50, key is test_row_0/C:col10/1732032981759/Put/seqid=0 2024-11-19T16:16:23,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742294_1470 (size=12001) 2024-11-19T16:16:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T16:16:23,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T16:16:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:23,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:23,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033043357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:23,364 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 38dce676c0e0bc7474a9f822adbf177b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:23,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T16:16:23,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:23,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:23,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:23,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:23,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T16:16:23,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44532 deadline: 1732033043507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:23,515 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 38dce676c0e0bc7474a9f822adbf177b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:23,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033043597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:23,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T16:16:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:23,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:23,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033043636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:23,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033043636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:23,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/b317995e536044e0a9b01b053f623962 2024-11-19T16:16:23,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/f610728d5a364301a27e414995c7890c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/f610728d5a364301a27e414995c7890c 2024-11-19T16:16:23,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/f610728d5a364301a27e414995c7890c, entries=200, sequenceid=94, filesize=14.0 K 2024-11-19T16:16:23,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/7cd7283370524e9395397256d0cbc01b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/7cd7283370524e9395397256d0cbc01b 2024-11-19T16:16:23,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/7cd7283370524e9395397256d0cbc01b, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T16:16:23,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/b317995e536044e0a9b01b053f623962 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/b317995e536044e0a9b01b053f623962 2024-11-19T16:16:23,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/b317995e536044e0a9b01b053f623962, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T16:16:23,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 38dce676c0e0bc7474a9f822adbf177b in 1244ms, sequenceid=94, compaction requested=false 2024-11-19T16:16:23,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:23,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T16:16:23,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:23,781 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:16:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:23,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/f1f053ad70314950b79d4e34eeb8c44a is 50, key is test_row_0/A:col10/1732032982447/Put/seqid=0 2024-11-19T16:16:23,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742295_1471 (size=12001) 2024-11-19T16:16:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T16:16:24,196 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/f1f053ad70314950b79d4e34eeb8c44a 2024-11-19T16:16:24,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/1c1074f1faee4565892bfea1d986350b is 50, key is test_row_0/B:col10/1732032982447/Put/seqid=0 2024-11-19T16:16:24,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742296_1472 (size=12001) 2024-11-19T16:16:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T16:16:24,611 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/1c1074f1faee4565892bfea1d986350b 2024-11-19T16:16:24,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/9a3923cca9424116980743b075cebef8 is 50, key is test_row_0/C:col10/1732032982447/Put/seqid=0 2024-11-19T16:16:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:24,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:24,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742297_1473 (size=12001) 2024-11-19T16:16:24,632 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/9a3923cca9424116980743b075cebef8 2024-11-19T16:16:24,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/f1f053ad70314950b79d4e34eeb8c44a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/f1f053ad70314950b79d4e34eeb8c44a 2024-11-19T16:16:24,639 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/f1f053ad70314950b79d4e34eeb8c44a, entries=150, sequenceid=107, filesize=11.7 K 2024-11-19T16:16:24,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/1c1074f1faee4565892bfea1d986350b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1c1074f1faee4565892bfea1d986350b 2024-11-19T16:16:24,643 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1c1074f1faee4565892bfea1d986350b, entries=150, sequenceid=107, filesize=11.7 K 2024-11-19T16:16:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/9a3923cca9424116980743b075cebef8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/9a3923cca9424116980743b075cebef8 2024-11-19T16:16:24,646 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/9a3923cca9424116980743b075cebef8, entries=150, sequenceid=107, filesize=11.7 K 2024-11-19T16:16:24,648 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=33.54 KB/34350 for 38dce676c0e0bc7474a9f822adbf177b in 867ms, sequenceid=107, compaction requested=true 2024-11-19T16:16:24,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:24,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:24,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-19T16:16:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-19T16:16:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:24,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:16:24,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:24,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:24,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:24,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:24,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:24,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:24,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-19T16:16:24,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-11-19T16:16:24,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.4850 sec 2024-11-19T16:16:24,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/701d9e2eb06e476a84d690421cbaba95 is 50, key is test_row_0/A:col10/1732032984641/Put/seqid=0 2024-11-19T16:16:24,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742298_1474 (size=16681) 2024-11-19T16:16:24,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032984693, max timestamp=1732032984708Num uniques 859; 2024-11-19T16:16:24,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032984693, max timestamp=1732032984708Num uniques 859; 2024-11-19T16:16:24,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732032984693, max timestamp=1732032984708Num uniques 859; 2024-11-19T16:16:24,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:24,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:24,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:24,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033044765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:24,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033044766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:24,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033044767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:24,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033044877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:24,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033044877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:24,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033044886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:25,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/701d9e2eb06e476a84d690421cbaba95 2024-11-19T16:16:25,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/1a97b08cb6ea4574b6d47768f3e2fdc2 is 50, key is test_row_0/B:col10/1732032984641/Put/seqid=0 2024-11-19T16:16:25,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742299_1475 (size=12001) 2024-11-19T16:16:25,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:25,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033045097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:25,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:25,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033045097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:25,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:25,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:25,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033045107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T16:16:25,317 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:25,317 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-19T16:16:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T16:16:25,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:25,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:25,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:25,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033045416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:25,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033045417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T16:16:25,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033045427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:25,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:25,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/1a97b08cb6ea4574b6d47768f3e2fdc2 2024-11-19T16:16:25,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T16:16:25,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:25,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:25,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:25,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:25,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:25,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/a2a0257d49e748738c89313c4f7e92da is 50, key is test_row_0/C:col10/1732032984641/Put/seqid=0 2024-11-19T16:16:25,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742300_1476 (size=12001) 2024-11-19T16:16:25,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44532 deadline: 1732033045527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:25,538 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 38dce676c0e0bc7474a9f822adbf177b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:25,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T16:16:25,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:25,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:25,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:25,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:25,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T16:16:25,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T16:16:25,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:25,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:25,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:25,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:25,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:25,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/a2a0257d49e748738c89313c4f7e92da 2024-11-19T16:16:25,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/701d9e2eb06e476a84d690421cbaba95 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/701d9e2eb06e476a84d690421cbaba95 2024-11-19T16:16:25,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/701d9e2eb06e476a84d690421cbaba95, entries=250, sequenceid=117, filesize=16.3 K 2024-11-19T16:16:25,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/1a97b08cb6ea4574b6d47768f3e2fdc2 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1a97b08cb6ea4574b6d47768f3e2fdc2 2024-11-19T16:16:25,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1a97b08cb6ea4574b6d47768f3e2fdc2, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T16:16:25,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/a2a0257d49e748738c89313c4f7e92da as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/a2a0257d49e748738c89313c4f7e92da 2024-11-19T16:16:25,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/a2a0257d49e748738c89313c4f7e92da, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T16:16:25,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 38dce676c0e0bc7474a9f822adbf177b in 1244ms, sequenceid=117, compaction requested=true 2024-11-19T16:16:25,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:25,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:25,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:25,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:25,894 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:25,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:25,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:25,894 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:25,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:25,895 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55127 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:25,895 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/A is initiating minor compaction (all files) 2024-11-19T16:16:25,895 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/A in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:25,895 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/220bbb749bdf499bb0c35d69d4db59ec, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/f610728d5a364301a27e414995c7890c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/f1f053ad70314950b79d4e34eeb8c44a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/701d9e2eb06e476a84d690421cbaba95] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=53.8 K 2024-11-19T16:16:25,895 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:25,895 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/B is initiating minor compaction (all files) 2024-11-19T16:16:25,895 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/B in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:25,895 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 220bbb749bdf499bb0c35d69d4db59ec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032981306 2024-11-19T16:16:25,895 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4138b58532f74ae1847d5d0e525e1afc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/7cd7283370524e9395397256d0cbc01b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1c1074f1faee4565892bfea1d986350b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1a97b08cb6ea4574b6d47768f3e2fdc2] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=47.0 K 2024-11-19T16:16:25,896 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting f610728d5a364301a27e414995c7890c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732032981759 2024-11-19T16:16:25,896 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 4138b58532f74ae1847d5d0e525e1afc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032981306 2024-11-19T16:16:25,896 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1f053ad70314950b79d4e34eeb8c44a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732032982435 2024-11-19T16:16:25,896 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cd7283370524e9395397256d0cbc01b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732032981759 2024-11-19T16:16:25,896 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 701d9e2eb06e476a84d690421cbaba95, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732032984630 2024-11-19T16:16:25,896 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c1074f1faee4565892bfea1d986350b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732032982435 2024-11-19T16:16:25,896 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a97b08cb6ea4574b6d47768f3e2fdc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732032984641 2024-11-19T16:16:25,902 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#A#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:25,903 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/55a8b6129bc24919997e642beeb415f8 is 50, key is test_row_0/A:col10/1732032984641/Put/seqid=0 2024-11-19T16:16:25,903 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#B#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:25,903 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/d98eae75785544f5af7d056a5e718b73 is 50, key is test_row_0/B:col10/1732032984641/Put/seqid=0 2024-11-19T16:16:25,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742301_1477 (size=12241) 2024-11-19T16:16:25,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742302_1478 (size=12241) 2024-11-19T16:16:25,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T16:16:25,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:25,929 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T16:16:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:25,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/bb016123e3a744e0ae55815838bb52ec is 50, key is test_row_0/A:col10/1732032984753/Put/seqid=0 2024-11-19T16:16:25,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742303_1479 (size=12151) 2024-11-19T16:16:25,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:25,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T16:16:26,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033046006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:26,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:26,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033046006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033046006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:26,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033046136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:26,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033046136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:26,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033046136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:26,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:26,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:26,310 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/55a8b6129bc24919997e642beeb415f8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/55a8b6129bc24919997e642beeb415f8 2024-11-19T16:16:26,310 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/d98eae75785544f5af7d056a5e718b73 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/d98eae75785544f5af7d056a5e718b73 2024-11-19T16:16:26,313 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/A of 38dce676c0e0bc7474a9f822adbf177b into 55a8b6129bc24919997e642beeb415f8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:26,313 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/B of 38dce676c0e0bc7474a9f822adbf177b into d98eae75785544f5af7d056a5e718b73(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:26,313 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:26,313 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:26,313 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/A, priority=12, startTime=1732032985894; duration=0sec 2024-11-19T16:16:26,313 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/B, priority=12, startTime=1732032985894; duration=0sec 2024-11-19T16:16:26,313 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:26,313 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:26,313 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:A 2024-11-19T16:16:26,313 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:B 2024-11-19T16:16:26,313 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:26,314 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:26,314 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/C is initiating minor compaction (all files) 2024-11-19T16:16:26,315 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/C in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:26,315 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/66c7127e019b4de08d0267842dd3cf50, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/b317995e536044e0a9b01b053f623962, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/9a3923cca9424116980743b075cebef8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/a2a0257d49e748738c89313c4f7e92da] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=47.0 K 2024-11-19T16:16:26,315 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66c7127e019b4de08d0267842dd3cf50, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732032981306 2024-11-19T16:16:26,315 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting b317995e536044e0a9b01b053f623962, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732032981759 2024-11-19T16:16:26,315 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a3923cca9424116980743b075cebef8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732032982435 2024-11-19T16:16:26,316 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2a0257d49e748738c89313c4f7e92da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732032984641 2024-11-19T16:16:26,321 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#C#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:26,321 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/ce4032d1095a426ea6780f89483bfbce is 50, key is test_row_0/C:col10/1732032984641/Put/seqid=0 2024-11-19T16:16:26,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742304_1480 (size=12241) 2024-11-19T16:16:26,338 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/bb016123e3a744e0ae55815838bb52ec 2024-11-19T16:16:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/4932dc92fbca436abb0f6d298254ef29 is 50, key is test_row_0/B:col10/1732032984753/Put/seqid=0 2024-11-19T16:16:26,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742305_1481 (size=12151) 2024-11-19T16:16:26,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033046356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:26,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033046357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:26,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:26,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033046357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:26,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T16:16:26,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033046676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:26,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033046676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:26,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033046676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:26,728 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/ce4032d1095a426ea6780f89483bfbce as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/ce4032d1095a426ea6780f89483bfbce 2024-11-19T16:16:26,731 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/C of 38dce676c0e0bc7474a9f822adbf177b into ce4032d1095a426ea6780f89483bfbce(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:26,731 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:26,731 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/C, priority=12, startTime=1732032985894; duration=0sec 2024-11-19T16:16:26,731 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:26,731 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:C 2024-11-19T16:16:26,766 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/4932dc92fbca436abb0f6d298254ef29 2024-11-19T16:16:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/bf61bc20f04642de8160c65f5e40118f is 50, key is test_row_0/C:col10/1732032984753/Put/seqid=0 2024-11-19T16:16:26,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742306_1482 (size=12151) 2024-11-19T16:16:27,176 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/bf61bc20f04642de8160c65f5e40118f 2024-11-19T16:16:27,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/bb016123e3a744e0ae55815838bb52ec as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/bb016123e3a744e0ae55815838bb52ec 2024-11-19T16:16:27,182 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/bb016123e3a744e0ae55815838bb52ec, entries=150, sequenceid=153, filesize=11.9 K 2024-11-19T16:16:27,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/4932dc92fbca436abb0f6d298254ef29 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4932dc92fbca436abb0f6d298254ef29 2024-11-19T16:16:27,185 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4932dc92fbca436abb0f6d298254ef29, entries=150, sequenceid=153, filesize=11.9 K 2024-11-19T16:16:27,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/bf61bc20f04642de8160c65f5e40118f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/bf61bc20f04642de8160c65f5e40118f 2024-11-19T16:16:27,189 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/bf61bc20f04642de8160c65f5e40118f, entries=150, sequenceid=153, filesize=11.9 K 2024-11-19T16:16:27,189 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=60.38 KB/61830 for 38dce676c0e0bc7474a9f822adbf177b in 1260ms, sequenceid=153, compaction requested=false 2024-11-19T16:16:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-19T16:16:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-19T16:16:27,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-19T16:16:27,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8720 sec 2024-11-19T16:16:27,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.8760 sec 2024-11-19T16:16:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:27,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T16:16:27,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:27,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:27,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:27,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:27,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:27,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:27,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/2acf3b336bb049ae9b5ee614f4ad8648 is 50, key is test_row_0/A:col10/1732032985957/Put/seqid=0 2024-11-19T16:16:27,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742307_1483 (size=14541) 2024-11-19T16:16:27,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/2acf3b336bb049ae9b5ee614f4ad8648 2024-11-19T16:16:27,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/fd3280b064ac45ed95bd8780d2036c1e is 50, key is test_row_0/B:col10/1732032985957/Put/seqid=0 2024-11-19T16:16:27,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742308_1484 (size=12151) 2024-11-19T16:16:27,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033047241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:27,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033047242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:27,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033047247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:27,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033047356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:27,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:27,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033047367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033047367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:27,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033047396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:27,404 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 38dce676c0e0bc7474a9f822adbf177b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T16:16:27,467 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:27,467 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-19T16:16:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T16:16:27,469 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:27,469 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:27,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:27,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033047567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T16:16:27,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033047577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:27,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033047577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:27,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T16:16:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:27,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:27,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/fd3280b064ac45ed95bd8780d2036c1e 2024-11-19T16:16:27,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/18b1712bf3204ec5836c28499605fc5b is 50, key is test_row_0/C:col10/1732032985957/Put/seqid=0 2024-11-19T16:16:27,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742309_1485 (size=12151) 2024-11-19T16:16:27,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T16:16:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:27,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T16:16:27,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033047886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:27,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033047887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:27,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:27,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:27,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033047897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:27,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T16:16:27,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:27,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:27,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:27,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:27,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:28,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/18b1712bf3204ec5836c28499605fc5b 2024-11-19T16:16:28,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/2acf3b336bb049ae9b5ee614f4ad8648 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/2acf3b336bb049ae9b5ee614f4ad8648 2024-11-19T16:16:28,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/2acf3b336bb049ae9b5ee614f4ad8648, entries=200, sequenceid=169, filesize=14.2 K 2024-11-19T16:16:28,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/fd3280b064ac45ed95bd8780d2036c1e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/fd3280b064ac45ed95bd8780d2036c1e 2024-11-19T16:16:28,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/fd3280b064ac45ed95bd8780d2036c1e, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T16:16:28,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/18b1712bf3204ec5836c28499605fc5b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/18b1712bf3204ec5836c28499605fc5b 2024-11-19T16:16:28,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/18b1712bf3204ec5836c28499605fc5b, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T16:16:28,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 38dce676c0e0bc7474a9f822adbf177b in 862ms, sequenceid=169, compaction requested=true 2024-11-19T16:16:28,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:28,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:28,062 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:28,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:28,063 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:28,063 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:28,064 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/B is initiating minor compaction (all files) 2024-11-19T16:16:28,064 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/B in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:28,064 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:28,064 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/d98eae75785544f5af7d056a5e718b73, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4932dc92fbca436abb0f6d298254ef29, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/fd3280b064ac45ed95bd8780d2036c1e] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=35.7 K 2024-11-19T16:16:28,064 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/A is initiating minor compaction (all files) 2024-11-19T16:16:28,064 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/A in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:28,064 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/55a8b6129bc24919997e642beeb415f8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/bb016123e3a744e0ae55815838bb52ec, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/2acf3b336bb049ae9b5ee614f4ad8648] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=38.0 K 2024-11-19T16:16:28,065 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting d98eae75785544f5af7d056a5e718b73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732032984641 2024-11-19T16:16:28,065 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55a8b6129bc24919997e642beeb415f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732032984641 2024-11-19T16:16:28,066 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb016123e3a744e0ae55815838bb52ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732032984753 2024-11-19T16:16:28,066 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 4932dc92fbca436abb0f6d298254ef29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732032984753 2024-11-19T16:16:28,067 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting fd3280b064ac45ed95bd8780d2036c1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732032985957 2024-11-19T16:16:28,067 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2acf3b336bb049ae9b5ee614f4ad8648, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732032985957 2024-11-19T16:16:28,073 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#B#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:28,073 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#A#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:28,073 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/9a638645efef42aba4c71bc6988f3e69 is 50, key is test_row_0/B:col10/1732032985957/Put/seqid=0 2024-11-19T16:16:28,073 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/7eca4aaac71f4a8ba8fb1ee4b1186e6b is 50, key is test_row_0/A:col10/1732032985957/Put/seqid=0 2024-11-19T16:16:28,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T16:16:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:28,080 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T16:16:28,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:28,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:28,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:28,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:28,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:28,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T16:16:28,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742310_1486 (size=12493) 2024-11-19T16:16:28,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742311_1487 (size=12493) 2024-11-19T16:16:28,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/7fb6fe8b234f4ab28f5679db004740a0 is 50, key is test_row_0/A:col10/1732032987241/Put/seqid=0 2024-11-19T16:16:28,131 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/7eca4aaac71f4a8ba8fb1ee4b1186e6b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7eca4aaac71f4a8ba8fb1ee4b1186e6b 2024-11-19T16:16:28,137 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/A of 38dce676c0e0bc7474a9f822adbf177b into 7eca4aaac71f4a8ba8fb1ee4b1186e6b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:28,137 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:28,137 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/A, priority=13, startTime=1732032988062; duration=0sec 2024-11-19T16:16:28,137 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:28,137 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:A 2024-11-19T16:16:28,137 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:28,139 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:28,139 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/C is initiating minor compaction (all files) 2024-11-19T16:16:28,139 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/C in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:28,139 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/ce4032d1095a426ea6780f89483bfbce, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/bf61bc20f04642de8160c65f5e40118f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/18b1712bf3204ec5836c28499605fc5b] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=35.7 K 2024-11-19T16:16:28,139 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce4032d1095a426ea6780f89483bfbce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732032984641 2024-11-19T16:16:28,139 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf61bc20f04642de8160c65f5e40118f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732032984753 2024-11-19T16:16:28,140 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18b1712bf3204ec5836c28499605fc5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732032985957 2024-11-19T16:16:28,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742312_1488 (size=12151) 2024-11-19T16:16:28,151 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/7fb6fe8b234f4ab28f5679db004740a0 2024-11-19T16:16:28,164 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#C#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:28,165 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/8b24bab354ba423baea9fca393e8a9ac is 50, key is test_row_0/C:col10/1732032985957/Put/seqid=0 2024-11-19T16:16:28,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/405e45d81fdb402e9a318e4f06fd13a5 is 50, key is test_row_0/B:col10/1732032987241/Put/seqid=0 2024-11-19T16:16:28,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742313_1489 (size=12493) 2024-11-19T16:16:28,207 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/8b24bab354ba423baea9fca393e8a9ac as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/8b24bab354ba423baea9fca393e8a9ac 2024-11-19T16:16:28,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742314_1490 (size=12151) 2024-11-19T16:16:28,212 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/405e45d81fdb402e9a318e4f06fd13a5 2024-11-19T16:16:28,213 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/C of 38dce676c0e0bc7474a9f822adbf177b into 8b24bab354ba423baea9fca393e8a9ac(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:28,213 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:28,213 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/C, priority=13, startTime=1732032988063; duration=0sec 2024-11-19T16:16:28,213 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:28,214 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:C 2024-11-19T16:16:28,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/91dc5503323f42ec92ec2648f8aa7495 is 50, key is test_row_0/C:col10/1732032987241/Put/seqid=0 2024-11-19T16:16:28,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742315_1491 (size=12151) 2024-11-19T16:16:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:28,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033048420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:28,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033048423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:28,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033048428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:28,502 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/9a638645efef42aba4c71bc6988f3e69 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/9a638645efef42aba4c71bc6988f3e69 2024-11-19T16:16:28,505 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/B of 38dce676c0e0bc7474a9f822adbf177b into 9a638645efef42aba4c71bc6988f3e69(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:28,505 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:28,506 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/B, priority=13, startTime=1732032988062; duration=0sec 2024-11-19T16:16:28,506 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:28,506 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:B 2024-11-19T16:16:28,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033048537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033048537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:28,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033048537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:28,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T16:16:28,624 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/91dc5503323f42ec92ec2648f8aa7495 2024-11-19T16:16:28,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/7fb6fe8b234f4ab28f5679db004740a0 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7fb6fe8b234f4ab28f5679db004740a0 2024-11-19T16:16:28,631 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7fb6fe8b234f4ab28f5679db004740a0, entries=150, sequenceid=188, filesize=11.9 K 2024-11-19T16:16:28,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/405e45d81fdb402e9a318e4f06fd13a5 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/405e45d81fdb402e9a318e4f06fd13a5 2024-11-19T16:16:28,640 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/405e45d81fdb402e9a318e4f06fd13a5, entries=150, sequenceid=188, filesize=11.9 K 2024-11-19T16:16:28,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/91dc5503323f42ec92ec2648f8aa7495 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/91dc5503323f42ec92ec2648f8aa7495 2024-11-19T16:16:28,645 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/91dc5503323f42ec92ec2648f8aa7495, entries=150, sequenceid=188, filesize=11.9 K 2024-11-19T16:16:28,648 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 38dce676c0e0bc7474a9f822adbf177b in 568ms, sequenceid=188, compaction requested=false 2024-11-19T16:16:28,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:28,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:28,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-19T16:16:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-19T16:16:28,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.1840 sec 2024-11-19T16:16:28,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-19T16:16:28,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1800 sec 2024-11-19T16:16:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:28,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T16:16:28,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:28,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:28,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:28,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:28,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:28,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:28,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/8915451c3577467294164daaec8d3b08 is 50, key is test_row_0/A:col10/1732032988419/Put/seqid=0 2024-11-19T16:16:28,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742316_1492 (size=12151) 2024-11-19T16:16:28,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033048788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:28,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033048789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:28,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033048790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:28,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:28,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033048896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:28,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:28,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033048906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:28,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033048907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:29,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033049106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:29,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:29,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033049116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:29,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:29,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033049127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:29,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/8915451c3577467294164daaec8d3b08 2024-11-19T16:16:29,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/31e36b7d8bd5462eaaef8f54d78e5eb5 is 50, key is test_row_0/B:col10/1732032988419/Put/seqid=0 2024-11-19T16:16:29,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742317_1493 (size=12151) 2024-11-19T16:16:29,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033049417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:29,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:29,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033049436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:29,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033049447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:29,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/31e36b7d8bd5462eaaef8f54d78e5eb5 2024-11-19T16:16:29,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/9e0c9b35442646059b7d0b7e0d06efab is 50, key is test_row_0/C:col10/1732032988419/Put/seqid=0 2024-11-19T16:16:29,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44532 deadline: 1732033049577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:29,584 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 38dce676c0e0bc7474a9f822adbf177b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:29,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742318_1494 (size=12151) 2024-11-19T16:16:29,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/9e0c9b35442646059b7d0b7e0d06efab 2024-11-19T16:16:29,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/8915451c3577467294164daaec8d3b08 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/8915451c3577467294164daaec8d3b08 2024-11-19T16:16:29,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/8915451c3577467294164daaec8d3b08, entries=150, sequenceid=206, filesize=11.9 K 2024-11-19T16:16:29,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/31e36b7d8bd5462eaaef8f54d78e5eb5 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/31e36b7d8bd5462eaaef8f54d78e5eb5 2024-11-19T16:16:29,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/31e36b7d8bd5462eaaef8f54d78e5eb5, entries=150, sequenceid=206, filesize=11.9 K 2024-11-19T16:16:29,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/9e0c9b35442646059b7d0b7e0d06efab as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/9e0c9b35442646059b7d0b7e0d06efab 2024-11-19T16:16:29,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/9e0c9b35442646059b7d0b7e0d06efab, entries=150, sequenceid=206, filesize=11.9 K 2024-11-19T16:16:29,599 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T16:16:29,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 38dce676c0e0bc7474a9f822adbf177b in 847ms, sequenceid=206, compaction requested=true 2024-11-19T16:16:29,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:29,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:29,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:29,600 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:29,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:29,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:29,600 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:29,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:29,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:29,600 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:29,600 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:29,600 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/A is initiating minor compaction (all files) 2024-11-19T16:16:29,600 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/B is initiating minor compaction (all files) 2024-11-19T16:16:29,600 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/A in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:29,600 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/B in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:29,600 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7eca4aaac71f4a8ba8fb1ee4b1186e6b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7fb6fe8b234f4ab28f5679db004740a0, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/8915451c3577467294164daaec8d3b08] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=35.9 K 2024-11-19T16:16:29,600 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/9a638645efef42aba4c71bc6988f3e69, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/405e45d81fdb402e9a318e4f06fd13a5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/31e36b7d8bd5462eaaef8f54d78e5eb5] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=35.9 K 2024-11-19T16:16:29,601 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eca4aaac71f4a8ba8fb1ee4b1186e6b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732032985957 2024-11-19T16:16:29,601 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a638645efef42aba4c71bc6988f3e69, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732032985957 2024-11-19T16:16:29,601 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fb6fe8b234f4ab28f5679db004740a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732032987240 2024-11-19T16:16:29,601 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 405e45d81fdb402e9a318e4f06fd13a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732032987240 2024-11-19T16:16:29,601 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8915451c3577467294164daaec8d3b08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732032988419 2024-11-19T16:16:29,601 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 31e36b7d8bd5462eaaef8f54d78e5eb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732032988419 2024-11-19T16:16:29,606 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#A#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:29,606 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#B#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:29,607 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/b6f62cd703584405aea6c4c34910e0c6 is 50, key is test_row_0/A:col10/1732032988419/Put/seqid=0 2024-11-19T16:16:29,607 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/efd40f9c8dbe40b5ac8a9744e4ac14cc is 50, key is test_row_0/B:col10/1732032988419/Put/seqid=0 2024-11-19T16:16:29,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742319_1495 (size=12595) 2024-11-19T16:16:29,616 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/efd40f9c8dbe40b5ac8a9744e4ac14cc as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/efd40f9c8dbe40b5ac8a9744e4ac14cc 2024-11-19T16:16:29,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742320_1496 (size=12595) 2024-11-19T16:16:29,623 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/B of 38dce676c0e0bc7474a9f822adbf177b into efd40f9c8dbe40b5ac8a9744e4ac14cc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:29,623 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:29,623 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/B, priority=13, startTime=1732032989600; duration=0sec 2024-11-19T16:16:29,623 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:29,623 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:B 2024-11-19T16:16:29,623 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:29,624 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:29,624 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/C is initiating minor compaction (all files) 2024-11-19T16:16:29,624 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/C in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:29,624 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/8b24bab354ba423baea9fca393e8a9ac, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/91dc5503323f42ec92ec2648f8aa7495, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/9e0c9b35442646059b7d0b7e0d06efab] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=35.9 K 2024-11-19T16:16:29,624 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b24bab354ba423baea9fca393e8a9ac, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732032985957 2024-11-19T16:16:29,624 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 91dc5503323f42ec92ec2648f8aa7495, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732032987240 2024-11-19T16:16:29,624 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e0c9b35442646059b7d0b7e0d06efab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732032988419 2024-11-19T16:16:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T16:16:29,627 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:29,628 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-19T16:16:29,630 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:29,630 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#C#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:29,631 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:29,631 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/12e91f1f2e284eaeb37d4f54ef911c11 is 50, key is test_row_0/C:col10/1732032988419/Put/seqid=0 2024-11-19T16:16:29,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T16:16:29,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742321_1497 (size=12595) 2024-11-19T16:16:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T16:16:29,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T16:16:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:29,784 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T16:16:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:29,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/02d94f3662434b178b71444ebbe6cab5 is 50, key is test_row_0/A:col10/1732032988788/Put/seqid=0 2024-11-19T16:16:29,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742322_1498 (size=12151) 2024-11-19T16:16:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:29,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T16:16:29,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033049972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:29,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033049976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:29,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033049978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:30,021 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/b6f62cd703584405aea6c4c34910e0c6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/b6f62cd703584405aea6c4c34910e0c6 2024-11-19T16:16:30,024 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/A of 38dce676c0e0bc7474a9f822adbf177b into b6f62cd703584405aea6c4c34910e0c6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:30,024 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:30,024 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/A, priority=13, startTime=1732032989599; duration=0sec 2024-11-19T16:16:30,024 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:30,024 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:A 2024-11-19T16:16:30,039 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/12e91f1f2e284eaeb37d4f54ef911c11 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/12e91f1f2e284eaeb37d4f54ef911c11 2024-11-19T16:16:30,044 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/C of 38dce676c0e0bc7474a9f822adbf177b into 12e91f1f2e284eaeb37d4f54ef911c11(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:30,044 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:30,044 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/C, priority=13, startTime=1732032989600; duration=0sec 2024-11-19T16:16:30,044 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:30,044 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:C 2024-11-19T16:16:30,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033050096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:30,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033050096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033050096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:30,206 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/02d94f3662434b178b71444ebbe6cab5 2024-11-19T16:16:30,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/47f9dc9be30e474caf40129e3f4c7f1b is 50, key is test_row_0/B:col10/1732032988788/Put/seqid=0 2024-11-19T16:16:30,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742323_1499 (size=12151) 2024-11-19T16:16:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T16:16:30,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033050306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033050306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:30,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033050306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:30,616 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/47f9dc9be30e474caf40129e3f4c7f1b 2024-11-19T16:16:30,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033050617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:30,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033050617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:30,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033050617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:30,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/323eccf5dbb6463bb055928dd89e01e7 is 50, key is test_row_0/C:col10/1732032988788/Put/seqid=0 2024-11-19T16:16:30,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742324_1500 (size=12151) 2024-11-19T16:16:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T16:16:31,028 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/323eccf5dbb6463bb055928dd89e01e7 2024-11-19T16:16:31,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/02d94f3662434b178b71444ebbe6cab5 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/02d94f3662434b178b71444ebbe6cab5 2024-11-19T16:16:31,035 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/02d94f3662434b178b71444ebbe6cab5, entries=150, sequenceid=224, filesize=11.9 K 2024-11-19T16:16:31,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/47f9dc9be30e474caf40129e3f4c7f1b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/47f9dc9be30e474caf40129e3f4c7f1b 2024-11-19T16:16:31,038 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/47f9dc9be30e474caf40129e3f4c7f1b, entries=150, sequenceid=224, filesize=11.9 K 2024-11-19T16:16:31,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/323eccf5dbb6463bb055928dd89e01e7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/323eccf5dbb6463bb055928dd89e01e7 2024-11-19T16:16:31,041 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/323eccf5dbb6463bb055928dd89e01e7, entries=150, sequenceid=224, filesize=11.9 K 2024-11-19T16:16:31,042 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 38dce676c0e0bc7474a9f822adbf177b in 1258ms, sequenceid=224, compaction requested=false 2024-11-19T16:16:31,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:31,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:31,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-19T16:16:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-19T16:16:31,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-19T16:16:31,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-19T16:16:31,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4170 sec 2024-11-19T16:16:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:31,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T16:16:31,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:31,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:31,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:31,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:31,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:31,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:31,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/15c83db2ad084d97bb9f430c9ac87925 is 50, key is test_row_0/A:col10/1732032989956/Put/seqid=0 2024-11-19T16:16:31,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742325_1501 (size=14541) 2024-11-19T16:16:31,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/15c83db2ad084d97bb9f430c9ac87925 2024-11-19T16:16:31,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/48b0edc8c1e54dfea32ae292d395a2b1 is 50, key is test_row_0/B:col10/1732032989956/Put/seqid=0 2024-11-19T16:16:31,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742326_1502 (size=12151) 2024-11-19T16:16:31,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033051176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:31,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033051177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:31,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033051185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:31,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033051297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:31,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:31,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033051297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:31,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:31,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033051298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:31,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:31,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033051516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033051516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033051516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:31,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/48b0edc8c1e54dfea32ae292d395a2b1 2024-11-19T16:16:31,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/e5ceed720ca5441db9c5555cd17dcc88 is 50, key is test_row_0/C:col10/1732032989956/Put/seqid=0 2024-11-19T16:16:31,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742327_1503 (size=12151) 2024-11-19T16:16:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T16:16:31,777 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:31,778 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-19T16:16:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T16:16:31,779 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:31,780 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:31,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:31,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033051826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:31,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:31,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033051837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:31,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033051837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T16:16:31,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T16:16:31,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:31,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:31,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:31,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:31,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:31,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/e5ceed720ca5441db9c5555cd17dcc88 2024-11-19T16:16:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/15c83db2ad084d97bb9f430c9ac87925 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/15c83db2ad084d97bb9f430c9ac87925 2024-11-19T16:16:31,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/15c83db2ad084d97bb9f430c9ac87925, entries=200, sequenceid=242, filesize=14.2 K 2024-11-19T16:16:31,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/48b0edc8c1e54dfea32ae292d395a2b1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/48b0edc8c1e54dfea32ae292d395a2b1 2024-11-19T16:16:31,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/48b0edc8c1e54dfea32ae292d395a2b1, entries=150, sequenceid=242, filesize=11.9 K 2024-11-19T16:16:31,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/e5ceed720ca5441db9c5555cd17dcc88 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/e5ceed720ca5441db9c5555cd17dcc88 2024-11-19T16:16:31,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/e5ceed720ca5441db9c5555cd17dcc88, entries=150, sequenceid=242, filesize=11.9 K 2024-11-19T16:16:31,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 38dce676c0e0bc7474a9f822adbf177b in 844ms, sequenceid=242, compaction requested=true 2024-11-19T16:16:31,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:31,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:31,976 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:31,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:31,976 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:31,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:31,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:31,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:31,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:31,976 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:31,976 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:31,976 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/A is initiating minor compaction (all files) 2024-11-19T16:16:31,976 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/B is initiating minor compaction (all files) 2024-11-19T16:16:31,977 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/B in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:31,977 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/A in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:31,977 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/b6f62cd703584405aea6c4c34910e0c6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/02d94f3662434b178b71444ebbe6cab5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/15c83db2ad084d97bb9f430c9ac87925] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=38.4 K 2024-11-19T16:16:31,977 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/efd40f9c8dbe40b5ac8a9744e4ac14cc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/47f9dc9be30e474caf40129e3f4c7f1b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/48b0edc8c1e54dfea32ae292d395a2b1] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=36.0 K 2024-11-19T16:16:31,977 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting efd40f9c8dbe40b5ac8a9744e4ac14cc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732032988419 2024-11-19T16:16:31,977 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6f62cd703584405aea6c4c34910e0c6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732032988419 2024-11-19T16:16:31,977 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 47f9dc9be30e474caf40129e3f4c7f1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732032988761 2024-11-19T16:16:31,977 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02d94f3662434b178b71444ebbe6cab5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732032988761 2024-11-19T16:16:31,977 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 48b0edc8c1e54dfea32ae292d395a2b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732032989956 2024-11-19T16:16:31,978 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15c83db2ad084d97bb9f430c9ac87925, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732032989956 2024-11-19T16:16:31,983 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:31,983 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/4c2da77d1113441a86f7b583b3355e79 is 50, key is test_row_0/B:col10/1732032989956/Put/seqid=0 2024-11-19T16:16:31,986 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#A#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:31,986 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/fe256d1c8699410eac9d00af4a635a70 is 50, key is test_row_0/A:col10/1732032989956/Put/seqid=0 2024-11-19T16:16:31,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742328_1504 (size=12697) 2024-11-19T16:16:31,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742329_1505 (size=12697) 2024-11-19T16:16:31,993 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/fe256d1c8699410eac9d00af4a635a70 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/fe256d1c8699410eac9d00af4a635a70 2024-11-19T16:16:31,996 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/A of 38dce676c0e0bc7474a9f822adbf177b into fe256d1c8699410eac9d00af4a635a70(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:31,996 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:31,996 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/A, priority=13, startTime=1732032991975; duration=0sec 2024-11-19T16:16:31,997 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:31,997 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:A 2024-11-19T16:16:31,997 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:31,997 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:31,997 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/C is initiating minor compaction (all files) 2024-11-19T16:16:31,997 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/C in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:31,997 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/12e91f1f2e284eaeb37d4f54ef911c11, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/323eccf5dbb6463bb055928dd89e01e7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/e5ceed720ca5441db9c5555cd17dcc88] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=36.0 K 2024-11-19T16:16:31,998 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12e91f1f2e284eaeb37d4f54ef911c11, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732032988419 2024-11-19T16:16:31,998 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 323eccf5dbb6463bb055928dd89e01e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732032988761 2024-11-19T16:16:31,998 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5ceed720ca5441db9c5555cd17dcc88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732032989956 2024-11-19T16:16:32,003 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#C#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:32,004 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/1bc753f0d629443aa11a2386d08168f8 is 50, key is test_row_0/C:col10/1732032989956/Put/seqid=0 2024-11-19T16:16:32,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742330_1506 (size=12697) 2024-11-19T16:16:32,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T16:16:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:32,085 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T16:16:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:32,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/c2fe19dab2c14cd0b09dc5fee17d4d71 is 50, key is test_row_0/A:col10/1732032991159/Put/seqid=0 2024-11-19T16:16:32,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742331_1507 (size=12201) 2024-11-19T16:16:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T16:16:32,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:32,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033052377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:32,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033052381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:32,392 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/4c2da77d1113441a86f7b583b3355e79 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4c2da77d1113441a86f7b583b3355e79 2024-11-19T16:16:32,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033052387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:32,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:32,395 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/B of 38dce676c0e0bc7474a9f822adbf177b into 4c2da77d1113441a86f7b583b3355e79(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:32,395 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:32,395 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/B, priority=13, startTime=1732032991976; duration=0sec 2024-11-19T16:16:32,395 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:32,396 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:B 2024-11-19T16:16:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T16:16:32,409 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/1bc753f0d629443aa11a2386d08168f8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/1bc753f0d629443aa11a2386d08168f8 2024-11-19T16:16:32,415 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/C of 38dce676c0e0bc7474a9f822adbf177b into 1bc753f0d629443aa11a2386d08168f8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:32,415 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:32,415 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/C, priority=13, startTime=1732032991976; duration=0sec 2024-11-19T16:16:32,415 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:32,415 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:C 2024-11-19T16:16:32,492 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/c2fe19dab2c14cd0b09dc5fee17d4d71 2024-11-19T16:16:32,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/609be6460af74d24aca172712b5d35cc is 50, key is test_row_0/B:col10/1732032991159/Put/seqid=0 2024-11-19T16:16:32,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742332_1508 (size=12201) 2024-11-19T16:16:32,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033052496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:32,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033052496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033052497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:32,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033052716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:32,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033052716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:32,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033052717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:32,900 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/609be6460af74d24aca172712b5d35cc 2024-11-19T16:16:32,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/3e3604dfa8114d6dbbfb4aa7a410ca21 is 50, key is test_row_0/C:col10/1732032991159/Put/seqid=0 2024-11-19T16:16:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T16:16:32,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742333_1509 (size=12201) 2024-11-19T16:16:33,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033053027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:33,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:33,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033053027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:33,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:33,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033053027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:33,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:33,318 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/3e3604dfa8114d6dbbfb4aa7a410ca21 2024-11-19T16:16:33,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/c2fe19dab2c14cd0b09dc5fee17d4d71 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/c2fe19dab2c14cd0b09dc5fee17d4d71 2024-11-19T16:16:33,324 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/c2fe19dab2c14cd0b09dc5fee17d4d71, entries=150, sequenceid=259, filesize=11.9 K 2024-11-19T16:16:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/609be6460af74d24aca172712b5d35cc as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/609be6460af74d24aca172712b5d35cc 2024-11-19T16:16:33,328 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/609be6460af74d24aca172712b5d35cc, entries=150, sequenceid=259, filesize=11.9 K 2024-11-19T16:16:33,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/3e3604dfa8114d6dbbfb4aa7a410ca21 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/3e3604dfa8114d6dbbfb4aa7a410ca21 2024-11-19T16:16:33,332 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/3e3604dfa8114d6dbbfb4aa7a410ca21, entries=150, sequenceid=259, filesize=11.9 K 2024-11-19T16:16:33,332 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 38dce676c0e0bc7474a9f822adbf177b in 1247ms, sequenceid=259, compaction requested=false 2024-11-19T16:16:33,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:33,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:33,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-19T16:16:33,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-19T16:16:33,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-19T16:16:33,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5540 sec 2024-11-19T16:16:33,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5590 sec 2024-11-19T16:16:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:33,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T16:16:33,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:33,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:33,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:33,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:33,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:33,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:33,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/70cf17bbeae84724a2edb68f1a71b012 is 50, key is test_row_0/A:col10/1732032992375/Put/seqid=0 2024-11-19T16:16:33,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742334_1510 (size=14741) 2024-11-19T16:16:33,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/70cf17bbeae84724a2edb68f1a71b012 2024-11-19T16:16:33,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/19fb75af54c34518b1dcc55ce7ecc117 is 50, key is test_row_0/B:col10/1732032992375/Put/seqid=0 2024-11-19T16:16:33,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742335_1511 (size=12301) 2024-11-19T16:16:33,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/19fb75af54c34518b1dcc55ce7ecc117 2024-11-19T16:16:33,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/db6e7edd94d14e62a02c7deb946cf47c is 50, key is test_row_0/C:col10/1732032992375/Put/seqid=0 2024-11-19T16:16:33,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742336_1512 (size=12301) 2024-11-19T16:16:33,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/db6e7edd94d14e62a02c7deb946cf47c 2024-11-19T16:16:33,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/70cf17bbeae84724a2edb68f1a71b012 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/70cf17bbeae84724a2edb68f1a71b012 2024-11-19T16:16:33,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/70cf17bbeae84724a2edb68f1a71b012, entries=200, sequenceid=278, filesize=14.4 K 2024-11-19T16:16:33,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/19fb75af54c34518b1dcc55ce7ecc117 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/19fb75af54c34518b1dcc55ce7ecc117 2024-11-19T16:16:33,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033053587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:33,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:33,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/19fb75af54c34518b1dcc55ce7ecc117, entries=150, sequenceid=278, filesize=12.0 K 2024-11-19T16:16:33,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/db6e7edd94d14e62a02c7deb946cf47c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/db6e7edd94d14e62a02c7deb946cf47c 2024-11-19T16:16:33,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/db6e7edd94d14e62a02c7deb946cf47c, entries=150, sequenceid=278, filesize=12.0 K 2024-11-19T16:16:33,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 38dce676c0e0bc7474a9f822adbf177b in 55ms, sequenceid=278, compaction requested=true 2024-11-19T16:16:33,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:33,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:33,610 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:33,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:33,610 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:33,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:33,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:33,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:33,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:33,610 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39639 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:33,610 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:33,611 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/A is initiating minor compaction (all files) 2024-11-19T16:16:33,611 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/B is initiating minor compaction (all files) 2024-11-19T16:16:33,611 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/B in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:33,611 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/A in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:33,611 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4c2da77d1113441a86f7b583b3355e79, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/609be6460af74d24aca172712b5d35cc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/19fb75af54c34518b1dcc55ce7ecc117] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=36.3 K 2024-11-19T16:16:33,611 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/fe256d1c8699410eac9d00af4a635a70, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/c2fe19dab2c14cd0b09dc5fee17d4d71, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/70cf17bbeae84724a2edb68f1a71b012] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=38.7 K 2024-11-19T16:16:33,611 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c2da77d1113441a86f7b583b3355e79, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732032989956 2024-11-19T16:16:33,611 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe256d1c8699410eac9d00af4a635a70, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732032989956 2024-11-19T16:16:33,611 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2fe19dab2c14cd0b09dc5fee17d4d71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732032991159 2024-11-19T16:16:33,611 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 609be6460af74d24aca172712b5d35cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732032991159 2024-11-19T16:16:33,611 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 19fb75af54c34518b1dcc55ce7ecc117, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732032992375 2024-11-19T16:16:33,611 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70cf17bbeae84724a2edb68f1a71b012, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732032992373 2024-11-19T16:16:33,619 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#B#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:33,619 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#A#compaction#439 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:33,620 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/7e4ba29ff0c9468babf2a7f96d56b6fb is 50, key is test_row_0/A:col10/1732032992375/Put/seqid=0 2024-11-19T16:16:33,620 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/e972adddf86645a7bb581ee251720b59 is 50, key is test_row_0/B:col10/1732032992375/Put/seqid=0 2024-11-19T16:16:33,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T16:16:33,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:33,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:33,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:33,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:33,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:33,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:33,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742337_1513 (size=12949) 2024-11-19T16:16:33,629 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/7e4ba29ff0c9468babf2a7f96d56b6fb as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7e4ba29ff0c9468babf2a7f96d56b6fb 2024-11-19T16:16:33,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742338_1514 (size=12949) 2024-11-19T16:16:33,633 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/A of 38dce676c0e0bc7474a9f822adbf177b into 7e4ba29ff0c9468babf2a7f96d56b6fb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:33,633 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:33,633 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/A, priority=13, startTime=1732032993610; duration=0sec 2024-11-19T16:16:33,633 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:33,633 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:A 2024-11-19T16:16:33,634 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:33,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/fc4e78fa0a184343a9dcea26c528fcc3 is 50, key is test_row_0/A:col10/1732032993620/Put/seqid=0 2024-11-19T16:16:33,638 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:33,638 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/C is initiating minor compaction (all files) 2024-11-19T16:16:33,638 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/C in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:33,638 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/1bc753f0d629443aa11a2386d08168f8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/3e3604dfa8114d6dbbfb4aa7a410ca21, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/db6e7edd94d14e62a02c7deb946cf47c] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=36.3 K 2024-11-19T16:16:33,639 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bc753f0d629443aa11a2386d08168f8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732032989956 2024-11-19T16:16:33,639 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e3604dfa8114d6dbbfb4aa7a410ca21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732032991159 2024-11-19T16:16:33,639 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting db6e7edd94d14e62a02c7deb946cf47c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732032992375 2024-11-19T16:16:33,649 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#C#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:33,649 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/4e77bd19e9d746559bd1a4ece59f2d14 is 50, key is test_row_0/C:col10/1732032992375/Put/seqid=0 2024-11-19T16:16:33,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742339_1515 (size=14741) 2024-11-19T16:16:33,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742340_1516 (size=12949) 2024-11-19T16:16:33,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033053689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:33,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033053690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:33,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:33,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033053707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:33,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033053807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:33,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033053807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T16:16:33,927 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:33,928 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-19T16:16:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T16:16:33,929 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:33,929 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:33,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:33,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033053927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:33,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:33,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:34,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033054027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:34,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033054027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:34,034 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/e972adddf86645a7bb581ee251720b59 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/e972adddf86645a7bb581ee251720b59 2024-11-19T16:16:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T16:16:34,041 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/B of 38dce676c0e0bc7474a9f822adbf177b into e972adddf86645a7bb581ee251720b59(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:34,041 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:34,041 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/B, priority=13, startTime=1732032993610; duration=0sec 2024-11-19T16:16:34,042 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:34,042 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:B 2024-11-19T16:16:34,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/fc4e78fa0a184343a9dcea26c528fcc3 2024-11-19T16:16:34,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/b58a08a67cc842c188e4e2ddbc42d63b is 50, key is test_row_0/B:col10/1732032993620/Put/seqid=0 2024-11-19T16:16:34,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742341_1517 (size=12301) 2024-11-19T16:16:34,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/b58a08a67cc842c188e4e2ddbc42d63b 2024-11-19T16:16:34,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/b77a14b2519045b29804b411e494e2ff is 50, key is test_row_0/C:col10/1732032993620/Put/seqid=0 2024-11-19T16:16:34,074 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/4e77bd19e9d746559bd1a4ece59f2d14 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/4e77bd19e9d746559bd1a4ece59f2d14 2024-11-19T16:16:34,078 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/C of 38dce676c0e0bc7474a9f822adbf177b into 4e77bd19e9d746559bd1a4ece59f2d14(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:34,078 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:34,078 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/C, priority=13, startTime=1732032993610; duration=0sec 2024-11-19T16:16:34,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742342_1518 (size=12301) 2024-11-19T16:16:34,079 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:34,079 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:C 2024-11-19T16:16:34,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T16:16:34,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:34,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:34,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:34,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:34,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:34,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T16:16:34,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:34,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:34,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:34,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:34,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:34,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T16:16:34,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033054247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:34,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033054347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:34,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033054347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:34,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T16:16:34,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:34,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:34,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:34,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:34,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:34,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:34,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/b77a14b2519045b29804b411e494e2ff 2024-11-19T16:16:34,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/fc4e78fa0a184343a9dcea26c528fcc3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/fc4e78fa0a184343a9dcea26c528fcc3 2024-11-19T16:16:34,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/fc4e78fa0a184343a9dcea26c528fcc3, entries=200, sequenceid=296, filesize=14.4 K 2024-11-19T16:16:34,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/b58a08a67cc842c188e4e2ddbc42d63b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/b58a08a67cc842c188e4e2ddbc42d63b 2024-11-19T16:16:34,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/b58a08a67cc842c188e4e2ddbc42d63b, entries=150, sequenceid=296, filesize=12.0 K 2024-11-19T16:16:34,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/b77a14b2519045b29804b411e494e2ff as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/b77a14b2519045b29804b411e494e2ff 2024-11-19T16:16:34,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/b77a14b2519045b29804b411e494e2ff, entries=150, sequenceid=296, filesize=12.0 K 2024-11-19T16:16:34,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 38dce676c0e0bc7474a9f822adbf177b in 871ms, sequenceid=296, compaction requested=false 2024-11-19T16:16:34,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:34,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T16:16:34,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:34,540 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T16:16:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:34,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/ee21ab69b0eb442285b18e565a0d6bbf is 50, key is test_row_0/A:col10/1732032993662/Put/seqid=0 2024-11-19T16:16:34,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742343_1519 (size=12301) 2024-11-19T16:16:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T16:16:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:34,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:34,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033054862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:34,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033054878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:34,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033054880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:34,946 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/ee21ab69b0eb442285b18e565a0d6bbf 2024-11-19T16:16:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/ba96dfe4f5424c37964d7bd94208325f is 50, key is test_row_0/B:col10/1732032993662/Put/seqid=0 2024-11-19T16:16:34,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742344_1520 (size=12301) 2024-11-19T16:16:35,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033054996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:35,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033054996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T16:16:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033055206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:35,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033055217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:35,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:35,355 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/ba96dfe4f5424c37964d7bd94208325f 2024-11-19T16:16:35,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/403b3a13a13e45a48fe039e251f528c3 is 50, key is test_row_0/C:col10/1732032993662/Put/seqid=0 2024-11-19T16:16:35,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742345_1521 (size=12301) 2024-11-19T16:16:35,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033055517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:35,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:35,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033055527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:35,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:35,764 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/403b3a13a13e45a48fe039e251f528c3 2024-11-19T16:16:35,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/ee21ab69b0eb442285b18e565a0d6bbf as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/ee21ab69b0eb442285b18e565a0d6bbf 2024-11-19T16:16:35,774 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/ee21ab69b0eb442285b18e565a0d6bbf, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T16:16:35,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/ba96dfe4f5424c37964d7bd94208325f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/ba96dfe4f5424c37964d7bd94208325f 2024-11-19T16:16:35,779 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/ba96dfe4f5424c37964d7bd94208325f, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T16:16:35,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/403b3a13a13e45a48fe039e251f528c3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/403b3a13a13e45a48fe039e251f528c3 2024-11-19T16:16:35,785 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/403b3a13a13e45a48fe039e251f528c3, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T16:16:35,786 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 38dce676c0e0bc7474a9f822adbf177b in 1247ms, sequenceid=313, compaction requested=true 2024-11-19T16:16:35,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:35,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:35,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-19T16:16:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-19T16:16:35,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-19T16:16:35,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8590 sec 2024-11-19T16:16:35,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.8620 sec 2024-11-19T16:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:35,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T16:16:35,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:35,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:35,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:35,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:35,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:35,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:35,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/92e801abc8a54692abb7d60a6ded71e7 is 50, key is test_row_0/A:col10/1732032994878/Put/seqid=0 2024-11-19T16:16:35,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742346_1522 (size=14741) 2024-11-19T16:16:35,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033055971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:35,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:35,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:36,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033056026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:36,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:36,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:36,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:36,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033056047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T16:16:36,076 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:36,077 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-19T16:16:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T16:16:36,078 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:36,079 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:36,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:36,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033056086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:36,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:36,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:36,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T16:16:36,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T16:16:36,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:36,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:36,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:36,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033056296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:36,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/92e801abc8a54692abb7d60a6ded71e7 2024-11-19T16:16:36,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/afaaf8d746ed4ea08b5a8fdaf8baf366 is 50, key is test_row_0/B:col10/1732032994878/Put/seqid=0 2024-11-19T16:16:36,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742347_1523 (size=12301) 2024-11-19T16:16:36,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T16:16:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:36,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T16:16:36,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T16:16:36,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:36,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:36,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:36,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033056616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:36,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T16:16:36,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:36,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:36,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:36,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T16:16:36,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/afaaf8d746ed4ea08b5a8fdaf8baf366 2024-11-19T16:16:36,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/36b35201c55a4c59b12ec2e916a08747 is 50, key is test_row_0/C:col10/1732032994878/Put/seqid=0 2024-11-19T16:16:36,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742348_1524 (size=12301) 2024-11-19T16:16:36,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T16:16:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:36,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T16:16:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:36,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:37,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44484 deadline: 1732033057047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:37,053 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 38dce676c0e0bc7474a9f822adbf177b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:37,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44468 deadline: 1732033057067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:37,078 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 38dce676c0e0bc7474a9f822adbf177b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:37,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033057127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:37,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T16:16:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:37,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:37,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/36b35201c55a4c59b12ec2e916a08747 2024-11-19T16:16:37,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/92e801abc8a54692abb7d60a6ded71e7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/92e801abc8a54692abb7d60a6ded71e7 2024-11-19T16:16:37,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/92e801abc8a54692abb7d60a6ded71e7, entries=200, sequenceid=332, filesize=14.4 K 2024-11-19T16:16:37,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/afaaf8d746ed4ea08b5a8fdaf8baf366 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/afaaf8d746ed4ea08b5a8fdaf8baf366 2024-11-19T16:16:37,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/afaaf8d746ed4ea08b5a8fdaf8baf366, entries=150, sequenceid=332, filesize=12.0 K 2024-11-19T16:16:37,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/36b35201c55a4c59b12ec2e916a08747 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/36b35201c55a4c59b12ec2e916a08747 2024-11-19T16:16:37,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/36b35201c55a4c59b12ec2e916a08747, entries=150, sequenceid=332, filesize=12.0 K 2024-11-19T16:16:37,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 38dce676c0e0bc7474a9f822adbf177b in 1281ms, sequenceid=332, compaction requested=true 2024-11-19T16:16:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:37,181 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:37,181 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:37,184 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:37,184 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:37,184 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/B is initiating minor compaction (all files) 2024-11-19T16:16:37,184 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/A is initiating minor compaction (all files) 2024-11-19T16:16:37,184 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/B in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:37,184 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/A in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:37,184 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/e972adddf86645a7bb581ee251720b59, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/b58a08a67cc842c188e4e2ddbc42d63b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/ba96dfe4f5424c37964d7bd94208325f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/afaaf8d746ed4ea08b5a8fdaf8baf366] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=48.7 K 2024-11-19T16:16:37,184 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7e4ba29ff0c9468babf2a7f96d56b6fb, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/fc4e78fa0a184343a9dcea26c528fcc3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/ee21ab69b0eb442285b18e565a0d6bbf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/92e801abc8a54692abb7d60a6ded71e7] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=53.4 K 2024-11-19T16:16:37,184 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting e972adddf86645a7bb581ee251720b59, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732032992375 2024-11-19T16:16:37,184 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e4ba29ff0c9468babf2a7f96d56b6fb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732032992375 2024-11-19T16:16:37,185 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc4e78fa0a184343a9dcea26c528fcc3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732032993585 2024-11-19T16:16:37,185 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting b58a08a67cc842c188e4e2ddbc42d63b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732032993585 2024-11-19T16:16:37,185 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee21ab69b0eb442285b18e565a0d6bbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732032993662 2024-11-19T16:16:37,185 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting ba96dfe4f5424c37964d7bd94208325f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732032993662 2024-11-19T16:16:37,185 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92e801abc8a54692abb7d60a6ded71e7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732032994774 2024-11-19T16:16:37,185 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting afaaf8d746ed4ea08b5a8fdaf8baf366, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732032994839 2024-11-19T16:16:37,191 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:37,191 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/268ff2a674174fd7a2ec8956fa558778 is 50, key is test_row_0/B:col10/1732032994878/Put/seqid=0 2024-11-19T16:16:37,191 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#A#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:37,192 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/551a9137e38e47b2adfbfda12cd4a94e is 50, key is test_row_0/A:col10/1732032994878/Put/seqid=0 2024-11-19T16:16:37,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742349_1525 (size=13085) 2024-11-19T16:16:37,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742350_1526 (size=13085) 2024-11-19T16:16:37,206 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/551a9137e38e47b2adfbfda12cd4a94e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/551a9137e38e47b2adfbfda12cd4a94e 2024-11-19T16:16:37,209 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/268ff2a674174fd7a2ec8956fa558778 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/268ff2a674174fd7a2ec8956fa558778 2024-11-19T16:16:37,209 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/A of 38dce676c0e0bc7474a9f822adbf177b into 551a9137e38e47b2adfbfda12cd4a94e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:37,209 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:37,209 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/A, priority=12, startTime=1732032997181; duration=0sec 2024-11-19T16:16:37,210 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:37,210 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:A 2024-11-19T16:16:37,210 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:37,210 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:37,210 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/C is initiating minor compaction (all files) 2024-11-19T16:16:37,210 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/C in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:37,210 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/4e77bd19e9d746559bd1a4ece59f2d14, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/b77a14b2519045b29804b411e494e2ff, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/403b3a13a13e45a48fe039e251f528c3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/36b35201c55a4c59b12ec2e916a08747] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=48.7 K 2024-11-19T16:16:37,211 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e77bd19e9d746559bd1a4ece59f2d14, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732032992375 2024-11-19T16:16:37,211 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting b77a14b2519045b29804b411e494e2ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732032993585 2024-11-19T16:16:37,211 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 403b3a13a13e45a48fe039e251f528c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732032993662 2024-11-19T16:16:37,211 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36b35201c55a4c59b12ec2e916a08747, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732032994839 2024-11-19T16:16:37,212 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/B of 38dce676c0e0bc7474a9f822adbf177b into 268ff2a674174fd7a2ec8956fa558778(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:37,212 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:37,212 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/B, priority=12, startTime=1732032997181; duration=0sec 2024-11-19T16:16:37,212 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:37,212 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:B 2024-11-19T16:16:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T16:16:37,217 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#C#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:37,218 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/0998099cde76453fb3700da75b1f9d04 is 50, key is test_row_0/C:col10/1732032994878/Put/seqid=0 2024-11-19T16:16:37,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742351_1527 (size=13085) 2024-11-19T16:16:37,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T16:16:37,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:37,299 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T16:16:37,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:37,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:37,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:37,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:37,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:37,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:37,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/441075204fae4dffb6a9cbb98bbb6272 is 50, key is test_row_0/A:col10/1732032995962/Put/seqid=0 2024-11-19T16:16:37,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742352_1528 (size=12301) 2024-11-19T16:16:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:37,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:37,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033057553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:37,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:37,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:37,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:37,625 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/0998099cde76453fb3700da75b1f9d04 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/0998099cde76453fb3700da75b1f9d04 2024-11-19T16:16:37,628 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/C of 38dce676c0e0bc7474a9f822adbf177b into 0998099cde76453fb3700da75b1f9d04(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:37,628 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:37,628 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/C, priority=12, startTime=1732032997181; duration=0sec 2024-11-19T16:16:37,628 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:37,628 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:C 2024-11-19T16:16:37,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033057677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:37,707 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/441075204fae4dffb6a9cbb98bbb6272 2024-11-19T16:16:37,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/6a0a3686cce047a8abfaf5f4993ad01b is 50, key is test_row_0/B:col10/1732032995962/Put/seqid=0 2024-11-19T16:16:37,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742353_1529 (size=12301) 2024-11-19T16:16:37,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033057887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:38,115 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/6a0a3686cce047a8abfaf5f4993ad01b 2024-11-19T16:16:38,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/fbb227724f424059b87080a9e4819ac7 is 50, key is test_row_0/C:col10/1732032995962/Put/seqid=0 2024-11-19T16:16:38,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742354_1530 (size=12301) 2024-11-19T16:16:38,125 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/fbb227724f424059b87080a9e4819ac7 2024-11-19T16:16:38,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/441075204fae4dffb6a9cbb98bbb6272 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/441075204fae4dffb6a9cbb98bbb6272 2024-11-19T16:16:38,131 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/441075204fae4dffb6a9cbb98bbb6272, entries=150, sequenceid=347, filesize=12.0 K 2024-11-19T16:16:38,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/6a0a3686cce047a8abfaf5f4993ad01b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/6a0a3686cce047a8abfaf5f4993ad01b 2024-11-19T16:16:38,134 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/6a0a3686cce047a8abfaf5f4993ad01b, entries=150, sequenceid=347, filesize=12.0 K 2024-11-19T16:16:38,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/fbb227724f424059b87080a9e4819ac7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/fbb227724f424059b87080a9e4819ac7 2024-11-19T16:16:38,137 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/fbb227724f424059b87080a9e4819ac7, entries=150, sequenceid=347, filesize=12.0 K 2024-11-19T16:16:38,137 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 38dce676c0e0bc7474a9f822adbf177b in 838ms, sequenceid=347, compaction requested=false 2024-11-19T16:16:38,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:38,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:38,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-19T16:16:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-19T16:16:38,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-19T16:16:38,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0600 sec 2024-11-19T16:16:38,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.0630 sec 2024-11-19T16:16:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:38,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T16:16:38,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:38,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:38,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:38,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:38,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:38,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:38,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/65877463443f4c36b6573366fd1c2397 is 50, key is test_row_0/A:col10/1732032997552/Put/seqid=0 2024-11-19T16:16:38,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742355_1531 (size=14741) 2024-11-19T16:16:38,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:38,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033058206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:38,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033058207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T16:16:38,237 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:38,237 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:38,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-19T16:16:38,238 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:38,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T16:16:38,239 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:38,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:38,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033058316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:38,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033058316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T16:16:38,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T16:16:38,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:38,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:38,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:38,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:38,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:38,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033058527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:38,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033058527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:38,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T16:16:38,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:38,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:38,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:38,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:38,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T16:16:38,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/65877463443f4c36b6573366fd1c2397 2024-11-19T16:16:38,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/81da9793f5614799a07d6d78321cc1f1 is 50, key is test_row_0/B:col10/1732032997552/Put/seqid=0 2024-11-19T16:16:38,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742356_1532 (size=12301) 2024-11-19T16:16:38,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/81da9793f5614799a07d6d78321cc1f1 2024-11-19T16:16:38,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/2c027ccb70cb48d68a509019c7ae5fa5 is 50, key is test_row_0/C:col10/1732032997552/Put/seqid=0 2024-11-19T16:16:38,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742357_1533 (size=12301) 2024-11-19T16:16:38,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T16:16:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:38,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:38,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44504 deadline: 1732033058847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:38,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44466 deadline: 1732033058847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 , the old value is region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., hostname=32a8860bb9d6,43095,1732032854050, seqNum=2 because the exception is null or not the one we care about 2024-11-19T16:16:38,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T16:16:38,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:38,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T16:16:38,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/2c027ccb70cb48d68a509019c7ae5fa5 2024-11-19T16:16:38,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/65877463443f4c36b6573366fd1c2397 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/65877463443f4c36b6573366fd1c2397 2024-11-19T16:16:38,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/65877463443f4c36b6573366fd1c2397, entries=200, sequenceid=368, filesize=14.4 K 2024-11-19T16:16:38,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/81da9793f5614799a07d6d78321cc1f1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/81da9793f5614799a07d6d78321cc1f1 2024-11-19T16:16:38,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/81da9793f5614799a07d6d78321cc1f1, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T16:16:38,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/2c027ccb70cb48d68a509019c7ae5fa5 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/2c027ccb70cb48d68a509019c7ae5fa5 2024-11-19T16:16:38,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/2c027ccb70cb48d68a509019c7ae5fa5, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T16:16:38,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 38dce676c0e0bc7474a9f822adbf177b in 836ms, sequenceid=368, compaction requested=true 2024-11-19T16:16:38,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:38,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:38,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:38,989 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:38,990 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:38,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:38,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:38,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38dce676c0e0bc7474a9f822adbf177b:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:38,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:38,990 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:38,990 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/A is initiating minor compaction (all files) 2024-11-19T16:16:38,990 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/A in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:38,990 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/551a9137e38e47b2adfbfda12cd4a94e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/441075204fae4dffb6a9cbb98bbb6272, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/65877463443f4c36b6573366fd1c2397] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=39.2 K 2024-11-19T16:16:38,990 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:38,991 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/B is initiating minor compaction (all files) 2024-11-19T16:16:38,991 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/B in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:38,991 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/268ff2a674174fd7a2ec8956fa558778, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/6a0a3686cce047a8abfaf5f4993ad01b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/81da9793f5614799a07d6d78321cc1f1] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=36.8 K 2024-11-19T16:16:38,991 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 268ff2a674174fd7a2ec8956fa558778, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732032994839 2024-11-19T16:16:38,991 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 551a9137e38e47b2adfbfda12cd4a94e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732032994839 2024-11-19T16:16:38,992 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 441075204fae4dffb6a9cbb98bbb6272, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732032995934 2024-11-19T16:16:38,992 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a0a3686cce047a8abfaf5f4993ad01b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732032995934 2024-11-19T16:16:38,992 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65877463443f4c36b6573366fd1c2397, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732032997524 2024-11-19T16:16:38,992 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 81da9793f5614799a07d6d78321cc1f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732032997541 2024-11-19T16:16:38,999 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#B#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:38,999 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#A#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:38,999 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/1267d93c1fe940e8910c8ca4d3b7220e is 50, key is test_row_0/B:col10/1732032997552/Put/seqid=0 2024-11-19T16:16:38,999 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/ffb2051ffb884c37831e48383bc077a7 is 50, key is test_row_0/A:col10/1732032997552/Put/seqid=0 2024-11-19T16:16:39,000 INFO [Thread-2052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2052. 2024-11-19T16:16:39,000 DEBUG [Thread-2052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:39,000 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,000 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:39,002 INFO [Thread-2056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2056. 2024-11-19T16:16:39,002 DEBUG [Thread-2056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:39,002 DEBUG [Thread-2056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,002 DEBUG [Thread-2056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:39,002 INFO [Thread-2058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2058. 2024-11-19T16:16:39,002 DEBUG [Thread-2058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:39,002 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,003 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:39,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742358_1534 (size=13187) 2024-11-19T16:16:39,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T16:16:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:39,005 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T16:16:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:39,006 INFO [Thread-2054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2054. 2024-11-19T16:16:39,006 DEBUG [Thread-2054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:39,006 DEBUG [Thread-2054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,006 DEBUG [Thread-2054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:39,008 INFO [Thread-2050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2050. 2024-11-19T16:16:39,008 DEBUG [Thread-2050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:39,008 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,008 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:39,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742359_1535 (size=13187) 2024-11-19T16:16:39,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/922aecdf00da497c9fcbe2565bcceeac is 50, key is test_row_0/A:col10/1732032998206/Put/seqid=0 2024-11-19T16:16:39,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742360_1536 (size=12301) 2024-11-19T16:16:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:39,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. as already flushing 2024-11-19T16:16:39,059 INFO [Thread-2047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2047. 2024-11-19T16:16:39,059 DEBUG [Thread-2047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:39,059 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,060 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:39,100 INFO [Thread-2041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2041. 2024-11-19T16:16:39,100 DEBUG [Thread-2041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:39,100 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,101 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:39,358 INFO [Thread-2045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2045. 2024-11-19T16:16:39,358 INFO [Thread-2043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2043. 2024-11-19T16:16:39,358 DEBUG [Thread-2045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:39,358 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,358 DEBUG [Thread-2043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:39,358 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,358 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,358 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,358 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:39,358 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T16:16:39,413 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/922aecdf00da497c9fcbe2565bcceeac 2024-11-19T16:16:39,414 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/ffb2051ffb884c37831e48383bc077a7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/ffb2051ffb884c37831e48383bc077a7 2024-11-19T16:16:39,417 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/1267d93c1fe940e8910c8ca4d3b7220e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1267d93c1fe940e8910c8ca4d3b7220e 2024-11-19T16:16:39,419 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/A of 38dce676c0e0bc7474a9f822adbf177b into ffb2051ffb884c37831e48383bc077a7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:39,419 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:39,419 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/A, priority=13, startTime=1732032998989; duration=0sec 2024-11-19T16:16:39,420 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:39,420 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:A 2024-11-19T16:16:39,420 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:39,420 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:39,420 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 38dce676c0e0bc7474a9f822adbf177b/C is initiating minor compaction (all files) 2024-11-19T16:16:39,420 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38dce676c0e0bc7474a9f822adbf177b/C in TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:39,421 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/0998099cde76453fb3700da75b1f9d04, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/fbb227724f424059b87080a9e4819ac7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/2c027ccb70cb48d68a509019c7ae5fa5] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp, totalSize=36.8 K 2024-11-19T16:16:39,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/8846abff0fe04bc6aa333c3df84a3474 is 50, key is test_row_0/B:col10/1732032998206/Put/seqid=0 2024-11-19T16:16:39,421 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0998099cde76453fb3700da75b1f9d04, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732032994839 2024-11-19T16:16:39,421 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbb227724f424059b87080a9e4819ac7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732032995934 2024-11-19T16:16:39,421 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/B of 38dce676c0e0bc7474a9f822adbf177b into 1267d93c1fe940e8910c8ca4d3b7220e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:39,421 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c027ccb70cb48d68a509019c7ae5fa5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732032997541 2024-11-19T16:16:39,421 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:39,421 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/B, priority=13, startTime=1732032998990; duration=0sec 2024-11-19T16:16:39,421 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:39,421 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:B 2024-11-19T16:16:39,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742361_1537 (size=12301) 2024-11-19T16:16:39,424 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/8846abff0fe04bc6aa333c3df84a3474 2024-11-19T16:16:39,427 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38dce676c0e0bc7474a9f822adbf177b#C#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:39,427 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/21251a8504954dbfa625f14bd605015b is 50, key is test_row_0/C:col10/1732032997552/Put/seqid=0 2024-11-19T16:16:39,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/df56e57fa2b14eb385b585c50992031d is 50, key is test_row_0/C:col10/1732032998206/Put/seqid=0 2024-11-19T16:16:39,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742362_1538 (size=13187) 2024-11-19T16:16:39,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742363_1539 (size=12301) 2024-11-19T16:16:39,692 INFO [Thread-2039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2039. 2024-11-19T16:16:39,692 DEBUG [Thread-2039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:16:39,692 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,693 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:39,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:39,832 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/df56e57fa2b14eb385b585c50992031d 2024-11-19T16:16:39,838 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/21251a8504954dbfa625f14bd605015b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/21251a8504954dbfa625f14bd605015b 2024-11-19T16:16:39,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/922aecdf00da497c9fcbe2565bcceeac as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/922aecdf00da497c9fcbe2565bcceeac 2024-11-19T16:16:39,844 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38dce676c0e0bc7474a9f822adbf177b/C of 38dce676c0e0bc7474a9f822adbf177b into 21251a8504954dbfa625f14bd605015b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:39,844 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/922aecdf00da497c9fcbe2565bcceeac, entries=150, sequenceid=380, filesize=12.0 K 2024-11-19T16:16:39,844 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:39,844 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., storeName=38dce676c0e0bc7474a9f822adbf177b/C, priority=13, startTime=1732032998990; duration=0sec 2024-11-19T16:16:39,844 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:39,845 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38dce676c0e0bc7474a9f822adbf177b:C 2024-11-19T16:16:39,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/8846abff0fe04bc6aa333c3df84a3474 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/8846abff0fe04bc6aa333c3df84a3474 2024-11-19T16:16:39,849 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/8846abff0fe04bc6aa333c3df84a3474, entries=150, sequenceid=380, filesize=12.0 K 2024-11-19T16:16:39,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/df56e57fa2b14eb385b585c50992031d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/df56e57fa2b14eb385b585c50992031d 2024-11-19T16:16:39,853 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/df56e57fa2b14eb385b585c50992031d, entries=150, sequenceid=380, filesize=12.0 K 2024-11-19T16:16:39,854 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=33.54 KB/34350 for 38dce676c0e0bc7474a9f822adbf177b in 849ms, sequenceid=380, compaction requested=false 2024-11-19T16:16:39,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 38dce676c0e0bc7474a9f822adbf177b: 2024-11-19T16:16:39,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:39,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-19T16:16:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-19T16:16:39,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-19T16:16:39,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6170 sec 2024-11-19T16:16:39,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.6220 sec 2024-11-19T16:16:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T16:16:40,388 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:40,388 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T16:16:40,388 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-19T16:16:40,388 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-19T16:16:40,388 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1170 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3505 rows 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1162 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3483 rows 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1161 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3479 rows 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1161 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3481 rows 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1157 2024-11-19T16:16:40,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3467 rows 2024-11-19T16:16:40,389 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T16:16:40,390 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T16:16:40,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:40,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:40,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:16:40,392 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-19T16:16:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T16:16:40,397 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732033000397"}]},"ts":"1732033000397"} 2024-11-19T16:16:40,398 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T16:16:40,398 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T16:16:40,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T16:16:40,400 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38dce676c0e0bc7474a9f822adbf177b, UNASSIGN}] 2024-11-19T16:16:40,401 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38dce676c0e0bc7474a9f822adbf177b, UNASSIGN 2024-11-19T16:16:40,401 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=38dce676c0e0bc7474a9f822adbf177b, regionState=CLOSING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:40,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38dce676c0e0bc7474a9f822adbf177b, UNASSIGN because future has completed 2024-11-19T16:16:40,403 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T16:16:40,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:16:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T16:16:40,555 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:40,555 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T16:16:40,556 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 38dce676c0e0bc7474a9f822adbf177b, disabling compactions & flushes 2024-11-19T16:16:40,556 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:40,556 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:40,556 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. after waiting 0 ms 2024-11-19T16:16:40,556 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:40,556 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing 38dce676c0e0bc7474a9f822adbf177b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T16:16:40,557 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=A 2024-11-19T16:16:40,557 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:40,557 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=B 2024-11-19T16:16:40,557 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:40,557 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38dce676c0e0bc7474a9f822adbf177b, store=C 2024-11-19T16:16:40,557 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:40,565 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/ccb6119355024172aeda0e2a0106c8eb is 50, key is test_row_0/A:col10/1732032999357/Put/seqid=0 2024-11-19T16:16:40,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742364_1540 (size=12301) 2024-11-19T16:16:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T16:16:40,972 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/ccb6119355024172aeda0e2a0106c8eb 2024-11-19T16:16:40,985 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/e7de834181d147c69d3681e064bb64df is 50, key is test_row_0/B:col10/1732032999357/Put/seqid=0 2024-11-19T16:16:40,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742365_1541 (size=12301) 2024-11-19T16:16:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T16:16:41,392 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/e7de834181d147c69d3681e064bb64df 2024-11-19T16:16:41,436 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/0b68da6b507e433e98adbe8b2ec0a1e5 is 50, key is test_row_0/C:col10/1732032999357/Put/seqid=0 2024-11-19T16:16:41,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742366_1542 (size=12301) 2024-11-19T16:16:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T16:16:41,842 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/0b68da6b507e433e98adbe8b2ec0a1e5 2024-11-19T16:16:41,852 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/A/ccb6119355024172aeda0e2a0106c8eb as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/ccb6119355024172aeda0e2a0106c8eb 2024-11-19T16:16:41,857 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/ccb6119355024172aeda0e2a0106c8eb, entries=150, sequenceid=391, filesize=12.0 K 2024-11-19T16:16:41,858 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/B/e7de834181d147c69d3681e064bb64df as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/e7de834181d147c69d3681e064bb64df 2024-11-19T16:16:41,863 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/e7de834181d147c69d3681e064bb64df, entries=150, sequenceid=391, filesize=12.0 K 2024-11-19T16:16:41,863 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/.tmp/C/0b68da6b507e433e98adbe8b2ec0a1e5 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/0b68da6b507e433e98adbe8b2ec0a1e5 2024-11-19T16:16:41,867 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/0b68da6b507e433e98adbe8b2ec0a1e5, entries=150, sequenceid=391, filesize=12.0 K 2024-11-19T16:16:41,868 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 38dce676c0e0bc7474a9f822adbf177b in 1312ms, sequenceid=391, compaction requested=true 2024-11-19T16:16:41,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/d3c5bdf78399428a8ff1e85fd16b5f98, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/d0c60533027c4bcdb621695662a278af, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/81a329ccb88b4759bb3fd30aee401296, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/220bbb749bdf499bb0c35d69d4db59ec, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/f610728d5a364301a27e414995c7890c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/f1f053ad70314950b79d4e34eeb8c44a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/701d9e2eb06e476a84d690421cbaba95, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/55a8b6129bc24919997e642beeb415f8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/bb016123e3a744e0ae55815838bb52ec, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/2acf3b336bb049ae9b5ee614f4ad8648, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7eca4aaac71f4a8ba8fb1ee4b1186e6b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7fb6fe8b234f4ab28f5679db004740a0, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/b6f62cd703584405aea6c4c34910e0c6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/8915451c3577467294164daaec8d3b08, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/02d94f3662434b178b71444ebbe6cab5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/15c83db2ad084d97bb9f430c9ac87925, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/fe256d1c8699410eac9d00af4a635a70, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/c2fe19dab2c14cd0b09dc5fee17d4d71, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/70cf17bbeae84724a2edb68f1a71b012, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7e4ba29ff0c9468babf2a7f96d56b6fb, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/fc4e78fa0a184343a9dcea26c528fcc3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/ee21ab69b0eb442285b18e565a0d6bbf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/92e801abc8a54692abb7d60a6ded71e7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/551a9137e38e47b2adfbfda12cd4a94e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/441075204fae4dffb6a9cbb98bbb6272, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/65877463443f4c36b6573366fd1c2397] to archive 2024-11-19T16:16:41,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:16:41,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/d3c5bdf78399428a8ff1e85fd16b5f98 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/d3c5bdf78399428a8ff1e85fd16b5f98 2024-11-19T16:16:41,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/d0c60533027c4bcdb621695662a278af to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/d0c60533027c4bcdb621695662a278af 2024-11-19T16:16:41,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/81a329ccb88b4759bb3fd30aee401296 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/81a329ccb88b4759bb3fd30aee401296 2024-11-19T16:16:41,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/220bbb749bdf499bb0c35d69d4db59ec to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/220bbb749bdf499bb0c35d69d4db59ec 2024-11-19T16:16:41,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/f610728d5a364301a27e414995c7890c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/f610728d5a364301a27e414995c7890c 2024-11-19T16:16:41,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/f1f053ad70314950b79d4e34eeb8c44a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/f1f053ad70314950b79d4e34eeb8c44a 2024-11-19T16:16:41,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/701d9e2eb06e476a84d690421cbaba95 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/701d9e2eb06e476a84d690421cbaba95 2024-11-19T16:16:41,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/55a8b6129bc24919997e642beeb415f8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/55a8b6129bc24919997e642beeb415f8 2024-11-19T16:16:41,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/bb016123e3a744e0ae55815838bb52ec to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/bb016123e3a744e0ae55815838bb52ec 2024-11-19T16:16:41,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/2acf3b336bb049ae9b5ee614f4ad8648 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/2acf3b336bb049ae9b5ee614f4ad8648 2024-11-19T16:16:41,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7eca4aaac71f4a8ba8fb1ee4b1186e6b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7eca4aaac71f4a8ba8fb1ee4b1186e6b 2024-11-19T16:16:41,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7fb6fe8b234f4ab28f5679db004740a0 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7fb6fe8b234f4ab28f5679db004740a0 2024-11-19T16:16:41,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/b6f62cd703584405aea6c4c34910e0c6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/b6f62cd703584405aea6c4c34910e0c6 2024-11-19T16:16:41,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/8915451c3577467294164daaec8d3b08 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/8915451c3577467294164daaec8d3b08 2024-11-19T16:16:41,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/02d94f3662434b178b71444ebbe6cab5 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/02d94f3662434b178b71444ebbe6cab5 2024-11-19T16:16:41,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/15c83db2ad084d97bb9f430c9ac87925 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/15c83db2ad084d97bb9f430c9ac87925 2024-11-19T16:16:41,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/fe256d1c8699410eac9d00af4a635a70 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/fe256d1c8699410eac9d00af4a635a70 2024-11-19T16:16:41,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/c2fe19dab2c14cd0b09dc5fee17d4d71 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/c2fe19dab2c14cd0b09dc5fee17d4d71 2024-11-19T16:16:41,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/70cf17bbeae84724a2edb68f1a71b012 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/70cf17bbeae84724a2edb68f1a71b012 2024-11-19T16:16:41,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7e4ba29ff0c9468babf2a7f96d56b6fb to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/7e4ba29ff0c9468babf2a7f96d56b6fb 2024-11-19T16:16:41,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/fc4e78fa0a184343a9dcea26c528fcc3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/fc4e78fa0a184343a9dcea26c528fcc3 2024-11-19T16:16:41,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/ee21ab69b0eb442285b18e565a0d6bbf to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/ee21ab69b0eb442285b18e565a0d6bbf 2024-11-19T16:16:41,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/92e801abc8a54692abb7d60a6ded71e7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/92e801abc8a54692abb7d60a6ded71e7 2024-11-19T16:16:41,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/551a9137e38e47b2adfbfda12cd4a94e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/551a9137e38e47b2adfbfda12cd4a94e 2024-11-19T16:16:41,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/441075204fae4dffb6a9cbb98bbb6272 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/441075204fae4dffb6a9cbb98bbb6272 2024-11-19T16:16:41,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/65877463443f4c36b6573366fd1c2397 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/65877463443f4c36b6573366fd1c2397 2024-11-19T16:16:41,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/bc78110ff7734933a900a8c6e72ce6b1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/f2efb843bea845e489fd634f4886e972, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4138b58532f74ae1847d5d0e525e1afc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/b9a9585f47f04fadb8f50c6b5c17d45f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/7cd7283370524e9395397256d0cbc01b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1c1074f1faee4565892bfea1d986350b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/d98eae75785544f5af7d056a5e718b73, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1a97b08cb6ea4574b6d47768f3e2fdc2, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4932dc92fbca436abb0f6d298254ef29, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/9a638645efef42aba4c71bc6988f3e69, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/fd3280b064ac45ed95bd8780d2036c1e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/405e45d81fdb402e9a318e4f06fd13a5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/efd40f9c8dbe40b5ac8a9744e4ac14cc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/31e36b7d8bd5462eaaef8f54d78e5eb5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/47f9dc9be30e474caf40129e3f4c7f1b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4c2da77d1113441a86f7b583b3355e79, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/48b0edc8c1e54dfea32ae292d395a2b1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/609be6460af74d24aca172712b5d35cc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/e972adddf86645a7bb581ee251720b59, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/19fb75af54c34518b1dcc55ce7ecc117, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/b58a08a67cc842c188e4e2ddbc42d63b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/ba96dfe4f5424c37964d7bd94208325f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/268ff2a674174fd7a2ec8956fa558778, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/afaaf8d746ed4ea08b5a8fdaf8baf366, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/6a0a3686cce047a8abfaf5f4993ad01b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/81da9793f5614799a07d6d78321cc1f1] to archive 2024-11-19T16:16:41,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:16:41,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/bc78110ff7734933a900a8c6e72ce6b1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/bc78110ff7734933a900a8c6e72ce6b1 2024-11-19T16:16:41,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/f2efb843bea845e489fd634f4886e972 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/f2efb843bea845e489fd634f4886e972 2024-11-19T16:16:41,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4138b58532f74ae1847d5d0e525e1afc to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4138b58532f74ae1847d5d0e525e1afc 2024-11-19T16:16:41,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/b9a9585f47f04fadb8f50c6b5c17d45f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/b9a9585f47f04fadb8f50c6b5c17d45f 2024-11-19T16:16:41,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/7cd7283370524e9395397256d0cbc01b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/7cd7283370524e9395397256d0cbc01b 2024-11-19T16:16:41,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1c1074f1faee4565892bfea1d986350b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1c1074f1faee4565892bfea1d986350b 2024-11-19T16:16:41,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/d98eae75785544f5af7d056a5e718b73 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/d98eae75785544f5af7d056a5e718b73 2024-11-19T16:16:41,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1a97b08cb6ea4574b6d47768f3e2fdc2 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1a97b08cb6ea4574b6d47768f3e2fdc2 2024-11-19T16:16:41,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4932dc92fbca436abb0f6d298254ef29 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4932dc92fbca436abb0f6d298254ef29 2024-11-19T16:16:41,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/9a638645efef42aba4c71bc6988f3e69 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/9a638645efef42aba4c71bc6988f3e69 2024-11-19T16:16:41,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/fd3280b064ac45ed95bd8780d2036c1e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/fd3280b064ac45ed95bd8780d2036c1e 2024-11-19T16:16:41,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/405e45d81fdb402e9a318e4f06fd13a5 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/405e45d81fdb402e9a318e4f06fd13a5 2024-11-19T16:16:41,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/efd40f9c8dbe40b5ac8a9744e4ac14cc to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/efd40f9c8dbe40b5ac8a9744e4ac14cc 2024-11-19T16:16:41,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/31e36b7d8bd5462eaaef8f54d78e5eb5 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/31e36b7d8bd5462eaaef8f54d78e5eb5 2024-11-19T16:16:41,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/47f9dc9be30e474caf40129e3f4c7f1b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/47f9dc9be30e474caf40129e3f4c7f1b 2024-11-19T16:16:41,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4c2da77d1113441a86f7b583b3355e79 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/4c2da77d1113441a86f7b583b3355e79 2024-11-19T16:16:41,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/48b0edc8c1e54dfea32ae292d395a2b1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/48b0edc8c1e54dfea32ae292d395a2b1 2024-11-19T16:16:41,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/609be6460af74d24aca172712b5d35cc to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/609be6460af74d24aca172712b5d35cc 2024-11-19T16:16:41,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/e972adddf86645a7bb581ee251720b59 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/e972adddf86645a7bb581ee251720b59 2024-11-19T16:16:41,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/19fb75af54c34518b1dcc55ce7ecc117 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/19fb75af54c34518b1dcc55ce7ecc117 2024-11-19T16:16:41,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/b58a08a67cc842c188e4e2ddbc42d63b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/b58a08a67cc842c188e4e2ddbc42d63b 2024-11-19T16:16:41,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/ba96dfe4f5424c37964d7bd94208325f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/ba96dfe4f5424c37964d7bd94208325f 2024-11-19T16:16:41,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/268ff2a674174fd7a2ec8956fa558778 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/268ff2a674174fd7a2ec8956fa558778 2024-11-19T16:16:41,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/afaaf8d746ed4ea08b5a8fdaf8baf366 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/afaaf8d746ed4ea08b5a8fdaf8baf366 2024-11-19T16:16:41,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/6a0a3686cce047a8abfaf5f4993ad01b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/6a0a3686cce047a8abfaf5f4993ad01b 2024-11-19T16:16:41,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/81da9793f5614799a07d6d78321cc1f1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/81da9793f5614799a07d6d78321cc1f1 2024-11-19T16:16:41,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/bbb0e651b0f144b5bbaa667dc6019d71, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/6b84ba8f4feb417ca25d56a8977b9853, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/66c7127e019b4de08d0267842dd3cf50, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/d0fe68cbfbaa4602be716fb0592a61d8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/b317995e536044e0a9b01b053f623962, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/9a3923cca9424116980743b075cebef8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/ce4032d1095a426ea6780f89483bfbce, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/a2a0257d49e748738c89313c4f7e92da, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/bf61bc20f04642de8160c65f5e40118f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/8b24bab354ba423baea9fca393e8a9ac, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/18b1712bf3204ec5836c28499605fc5b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/91dc5503323f42ec92ec2648f8aa7495, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/12e91f1f2e284eaeb37d4f54ef911c11, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/9e0c9b35442646059b7d0b7e0d06efab, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/323eccf5dbb6463bb055928dd89e01e7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/1bc753f0d629443aa11a2386d08168f8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/e5ceed720ca5441db9c5555cd17dcc88, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/3e3604dfa8114d6dbbfb4aa7a410ca21, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/4e77bd19e9d746559bd1a4ece59f2d14, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/db6e7edd94d14e62a02c7deb946cf47c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/b77a14b2519045b29804b411e494e2ff, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/403b3a13a13e45a48fe039e251f528c3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/0998099cde76453fb3700da75b1f9d04, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/36b35201c55a4c59b12ec2e916a08747, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/fbb227724f424059b87080a9e4819ac7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/2c027ccb70cb48d68a509019c7ae5fa5] to archive 2024-11-19T16:16:41,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:16:41,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/bbb0e651b0f144b5bbaa667dc6019d71 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/bbb0e651b0f144b5bbaa667dc6019d71 2024-11-19T16:16:41,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/6b84ba8f4feb417ca25d56a8977b9853 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/6b84ba8f4feb417ca25d56a8977b9853 2024-11-19T16:16:41,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/66c7127e019b4de08d0267842dd3cf50 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/66c7127e019b4de08d0267842dd3cf50 2024-11-19T16:16:41,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/d0fe68cbfbaa4602be716fb0592a61d8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/d0fe68cbfbaa4602be716fb0592a61d8 2024-11-19T16:16:41,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/b317995e536044e0a9b01b053f623962 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/b317995e536044e0a9b01b053f623962 2024-11-19T16:16:41,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/9a3923cca9424116980743b075cebef8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/9a3923cca9424116980743b075cebef8 2024-11-19T16:16:41,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/ce4032d1095a426ea6780f89483bfbce to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/ce4032d1095a426ea6780f89483bfbce 2024-11-19T16:16:41,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/a2a0257d49e748738c89313c4f7e92da to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/a2a0257d49e748738c89313c4f7e92da 2024-11-19T16:16:41,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/bf61bc20f04642de8160c65f5e40118f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/bf61bc20f04642de8160c65f5e40118f 2024-11-19T16:16:41,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/8b24bab354ba423baea9fca393e8a9ac to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/8b24bab354ba423baea9fca393e8a9ac 2024-11-19T16:16:41,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/18b1712bf3204ec5836c28499605fc5b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/18b1712bf3204ec5836c28499605fc5b 2024-11-19T16:16:41,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/91dc5503323f42ec92ec2648f8aa7495 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/91dc5503323f42ec92ec2648f8aa7495 2024-11-19T16:16:41,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/12e91f1f2e284eaeb37d4f54ef911c11 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/12e91f1f2e284eaeb37d4f54ef911c11 2024-11-19T16:16:41,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/9e0c9b35442646059b7d0b7e0d06efab to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/9e0c9b35442646059b7d0b7e0d06efab 2024-11-19T16:16:41,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/323eccf5dbb6463bb055928dd89e01e7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/323eccf5dbb6463bb055928dd89e01e7 2024-11-19T16:16:41,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/1bc753f0d629443aa11a2386d08168f8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/1bc753f0d629443aa11a2386d08168f8 2024-11-19T16:16:41,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/e5ceed720ca5441db9c5555cd17dcc88 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/e5ceed720ca5441db9c5555cd17dcc88 2024-11-19T16:16:41,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/3e3604dfa8114d6dbbfb4aa7a410ca21 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/3e3604dfa8114d6dbbfb4aa7a410ca21 2024-11-19T16:16:41,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/4e77bd19e9d746559bd1a4ece59f2d14 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/4e77bd19e9d746559bd1a4ece59f2d14 2024-11-19T16:16:41,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/db6e7edd94d14e62a02c7deb946cf47c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/db6e7edd94d14e62a02c7deb946cf47c 2024-11-19T16:16:41,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/b77a14b2519045b29804b411e494e2ff to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/b77a14b2519045b29804b411e494e2ff 2024-11-19T16:16:41,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/403b3a13a13e45a48fe039e251f528c3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/403b3a13a13e45a48fe039e251f528c3 2024-11-19T16:16:41,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/0998099cde76453fb3700da75b1f9d04 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/0998099cde76453fb3700da75b1f9d04 2024-11-19T16:16:41,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/36b35201c55a4c59b12ec2e916a08747 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/36b35201c55a4c59b12ec2e916a08747 2024-11-19T16:16:41,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/fbb227724f424059b87080a9e4819ac7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/fbb227724f424059b87080a9e4819ac7 2024-11-19T16:16:41,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/2c027ccb70cb48d68a509019c7ae5fa5 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/2c027ccb70cb48d68a509019c7ae5fa5 2024-11-19T16:16:41,930 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/recovered.edits/394.seqid, newMaxSeqId=394, maxSeqId=1 2024-11-19T16:16:41,930 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. 2024-11-19T16:16:41,931 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 38dce676c0e0bc7474a9f822adbf177b: Waiting for close lock at 1732033000556Running coprocessor pre-close hooks at 1732033000556Disabling compacts and flushes for region at 1732033000556Disabling writes for close at 1732033000556Obtaining lock to block concurrent updates at 1732033000556Preparing flush snapshotting stores in 38dce676c0e0bc7474a9f822adbf177b at 1732033000556Finished memstore snapshotting TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732033000557 (+1 ms)Flushing stores of TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b. at 1732033000558 (+1 ms)Flushing 38dce676c0e0bc7474a9f822adbf177b/A: creating writer at 1732033000558Flushing 38dce676c0e0bc7474a9f822adbf177b/A: appending metadata at 1732033000564 (+6 ms)Flushing 38dce676c0e0bc7474a9f822adbf177b/A: closing flushed file at 1732033000564Flushing 38dce676c0e0bc7474a9f822adbf177b/B: creating writer at 1732033000982 (+418 ms)Flushing 38dce676c0e0bc7474a9f822adbf177b/B: appending metadata at 1732033000985 (+3 ms)Flushing 38dce676c0e0bc7474a9f822adbf177b/B: closing flushed file at 1732033000985Flushing 38dce676c0e0bc7474a9f822adbf177b/C: creating writer at 1732033001428 (+443 ms)Flushing 38dce676c0e0bc7474a9f822adbf177b/C: appending metadata at 1732033001435 (+7 ms)Flushing 38dce676c0e0bc7474a9f822adbf177b/C: closing flushed file at 1732033001435Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@535cf1fc: reopening flushed file at 1732033001851 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b1305c0: reopening flushed file at 1732033001857 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@300ec4e1: reopening flushed file at 1732033001863 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 38dce676c0e0bc7474a9f822adbf177b in 1312ms, sequenceid=391, compaction requested=true at 1732033001868 (+5 ms)Writing region close event to WAL at 1732033001928 (+60 ms)Running coprocessor post-close hooks at 1732033001930 (+2 ms)Closed at 1732033001930 2024-11-19T16:16:41,932 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:41,933 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=38dce676c0e0bc7474a9f822adbf177b, regionState=CLOSED 2024-11-19T16:16:41,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:16:41,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-19T16:16:41,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 38dce676c0e0bc7474a9f822adbf177b, server=32a8860bb9d6,43095,1732032854050 in 1.5310 sec 2024-11-19T16:16:41,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-19T16:16:41,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38dce676c0e0bc7474a9f822adbf177b, UNASSIGN in 1.5350 sec 2024-11-19T16:16:41,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-19T16:16:41,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5380 sec 2024-11-19T16:16:41,939 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732033001939"}]},"ts":"1732033001939"} 2024-11-19T16:16:41,940 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T16:16:41,940 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T16:16:41,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5490 sec 2024-11-19T16:16:42,301 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T16:16:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T16:16:42,548 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:42,549 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-19T16:16:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:42,554 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T16:16:42,555 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:42,558 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:42,562 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/recovered.edits] 2024-11-19T16:16:42,566 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/922aecdf00da497c9fcbe2565bcceeac to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/922aecdf00da497c9fcbe2565bcceeac 2024-11-19T16:16:42,568 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/ccb6119355024172aeda0e2a0106c8eb to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/ccb6119355024172aeda0e2a0106c8eb 2024-11-19T16:16:42,570 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/ffb2051ffb884c37831e48383bc077a7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/A/ffb2051ffb884c37831e48383bc077a7 2024-11-19T16:16:42,574 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1267d93c1fe940e8910c8ca4d3b7220e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/1267d93c1fe940e8910c8ca4d3b7220e 2024-11-19T16:16:42,576 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/8846abff0fe04bc6aa333c3df84a3474 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/8846abff0fe04bc6aa333c3df84a3474 2024-11-19T16:16:42,578 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/e7de834181d147c69d3681e064bb64df to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/B/e7de834181d147c69d3681e064bb64df 2024-11-19T16:16:42,582 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/0b68da6b507e433e98adbe8b2ec0a1e5 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/0b68da6b507e433e98adbe8b2ec0a1e5 2024-11-19T16:16:42,584 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/21251a8504954dbfa625f14bd605015b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/21251a8504954dbfa625f14bd605015b 2024-11-19T16:16:42,586 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/df56e57fa2b14eb385b585c50992031d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/C/df56e57fa2b14eb385b585c50992031d 2024-11-19T16:16:42,590 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/recovered.edits/394.seqid to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b/recovered.edits/394.seqid 2024-11-19T16:16:42,590 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/38dce676c0e0bc7474a9f822adbf177b 2024-11-19T16:16:42,590 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T16:16:42,593 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:42,596 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T16:16:42,599 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T16:16:42,601 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:42,601 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T16:16:42,602 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732033002601"}]},"ts":"9223372036854775807"} 2024-11-19T16:16:42,604 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T16:16:42,604 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 38dce676c0e0bc7474a9f822adbf177b, NAME => 'TestAcidGuarantees,,1732032976785.38dce676c0e0bc7474a9f822adbf177b.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T16:16:42,604 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T16:16:42,604 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732033002604"}]},"ts":"9223372036854775807"} 2024-11-19T16:16:42,606 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T16:16:42,607 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:42,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 58 msec 2024-11-19T16:16:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T16:16:42,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T16:16:42,673 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:42,684 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 251), OpenFileDescriptor=447 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=453 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=4063 (was 4057) - AvailableMemoryMB LEAK? - 2024-11-19T16:16:42,693 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=453, ProcessCount=11, AvailableMemoryMB=4062 2024-11-19T16:16:42,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T16:16:42,695 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T16:16:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:42,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T16:16:42,697 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:42,697 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-19T16:16:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T16:16:42,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T16:16:42,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742367_1543 (size=1061) 2024-11-19T16:16:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T16:16:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T16:16:43,108 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70 2024-11-19T16:16:43,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742368_1544 (size=53) 2024-11-19T16:16:43,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T16:16:43,519 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:16:43,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0dc20a97b5ebf3ba62589849ed17c46f, disabling compactions & flushes 2024-11-19T16:16:43,520 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:43,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:43,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. after waiting 0 ms 2024-11-19T16:16:43,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:43,520 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:43,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0dc20a97b5ebf3ba62589849ed17c46f: Waiting for close lock at 1732033003520Disabling compacts and flushes for region at 1732033003520Disabling writes for close at 1732033003520Writing region close event to WAL at 1732033003520Closed at 1732033003520 2024-11-19T16:16:43,523 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T16:16:43,524 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732033003523"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732033003523"}]},"ts":"1732033003523"} 2024-11-19T16:16:43,528 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T16:16:43,529 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T16:16:43,530 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732033003529"}]},"ts":"1732033003529"} 2024-11-19T16:16:43,531 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T16:16:43,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, ASSIGN}] 2024-11-19T16:16:43,533 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, ASSIGN 2024-11-19T16:16:43,534 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, ASSIGN; state=OFFLINE, location=32a8860bb9d6,43095,1732032854050; forceNewPlan=false, retain=false 2024-11-19T16:16:43,598 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T16:16:43,685 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=0dc20a97b5ebf3ba62589849ed17c46f, regionState=OPENING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:43,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, ASSIGN because future has completed 2024-11-19T16:16:43,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:16:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T16:16:43,850 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:43,850 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} 2024-11-19T16:16:43,850 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:43,851 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:16:43,851 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:43,851 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:43,853 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:43,856 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:16:43,856 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dc20a97b5ebf3ba62589849ed17c46f columnFamilyName A 2024-11-19T16:16:43,856 DEBUG [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:43,857 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.HStore(327): Store=0dc20a97b5ebf3ba62589849ed17c46f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:16:43,857 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:43,859 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:16:43,859 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dc20a97b5ebf3ba62589849ed17c46f columnFamilyName B 2024-11-19T16:16:43,859 DEBUG [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:43,860 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.HStore(327): Store=0dc20a97b5ebf3ba62589849ed17c46f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:16:43,860 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:43,862 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:16:43,862 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dc20a97b5ebf3ba62589849ed17c46f columnFamilyName C 2024-11-19T16:16:43,862 DEBUG [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:43,863 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.HStore(327): Store=0dc20a97b5ebf3ba62589849ed17c46f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:16:43,863 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:43,864 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:43,865 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:43,865 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:43,867 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:43,867 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:43,868 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T16:16:43,869 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:43,872 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T16:16:43,872 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 0dc20a97b5ebf3ba62589849ed17c46f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74118603, jitterRate=0.10445325076580048}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T16:16:43,872 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:43,873 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 0dc20a97b5ebf3ba62589849ed17c46f: Running coprocessor pre-open hook at 1732033003851Writing region info on filesystem at 1732033003851Initializing all the Stores at 1732033003852 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732033003852Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732033003853 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732033003853Cleaning up temporary data from old regions at 1732033003867 (+14 ms)Running coprocessor post-open hooks at 1732033003872 (+5 ms)Region opened successfully at 1732033003873 (+1 ms) 2024-11-19T16:16:43,874 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., pid=140, masterSystemTime=1732033003845 2024-11-19T16:16:43,876 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:43,876 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:43,877 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=0dc20a97b5ebf3ba62589849ed17c46f, regionState=OPEN, openSeqNum=2, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:43,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:16:43,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-19T16:16:43,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 in 190 msec 2024-11-19T16:16:43,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-19T16:16:43,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, ASSIGN in 351 msec 2024-11-19T16:16:43,885 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T16:16:43,885 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732033003885"}]},"ts":"1732033003885"} 2024-11-19T16:16:43,886 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T16:16:43,887 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T16:16:43,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1920 sec 2024-11-19T16:16:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T16:16:44,847 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:44,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:44,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:44,850 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:44,851 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@968331d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:44,855 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38739f94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:44,856 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:44,858 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:44,859 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T16:16:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 32a8860bb9d6,37355,1732032853237 2024-11-19T16:16:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@75e002d1 2024-11-19T16:16:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T16:16:44,860 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T16:16:44,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T16:16:44,861 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T16:16:44,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T16:16:44,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742369_1545 (size=1097) 2024-11-19T16:16:45,274 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T16:16:45,274 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T16:16:45,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T16:16:45,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, REOPEN/MOVE}] 2024-11-19T16:16:45,282 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, REOPEN/MOVE 2024-11-19T16:16:45,283 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0dc20a97b5ebf3ba62589849ed17c46f, regionState=CLOSING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:45,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, REOPEN/MOVE because future has completed 2024-11-19T16:16:45,285 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T16:16:45,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:16:45,438 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,438 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T16:16:45,438 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 0dc20a97b5ebf3ba62589849ed17c46f, disabling compactions & flushes 2024-11-19T16:16:45,439 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:45,439 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:45,439 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. after waiting 0 ms 2024-11-19T16:16:45,439 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:45,448 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T16:16:45,449 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:45,449 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 0dc20a97b5ebf3ba62589849ed17c46f: Waiting for close lock at 1732033005438Running coprocessor pre-close hooks at 1732033005438Disabling compacts and flushes for region at 1732033005438Disabling writes for close at 1732033005439 (+1 ms)Writing region close event to WAL at 1732033005440 (+1 ms)Running coprocessor post-close hooks at 1732033005449 (+9 ms)Closed at 1732033005449 2024-11-19T16:16:45,449 WARN [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 0dc20a97b5ebf3ba62589849ed17c46f to self. 2024-11-19T16:16:45,451 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,452 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0dc20a97b5ebf3ba62589849ed17c46f, regionState=CLOSED 2024-11-19T16:16:45,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:16:45,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-19T16:16:45,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 in 170 msec 2024-11-19T16:16:45,459 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, REOPEN/MOVE; state=CLOSED, location=32a8860bb9d6,43095,1732032854050; forceNewPlan=false, retain=true 2024-11-19T16:16:45,610 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0dc20a97b5ebf3ba62589849ed17c46f, regionState=OPENING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:45,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, REOPEN/MOVE because future has completed 2024-11-19T16:16:45,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:16:45,769 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:45,769 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} 2024-11-19T16:16:45,769 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,769 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T16:16:45,769 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,769 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,771 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,771 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:16:45,772 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dc20a97b5ebf3ba62589849ed17c46f columnFamilyName A 2024-11-19T16:16:45,773 DEBUG [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:45,773 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.HStore(327): Store=0dc20a97b5ebf3ba62589849ed17c46f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:16:45,774 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,774 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:16:45,775 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dc20a97b5ebf3ba62589849ed17c46f columnFamilyName B 2024-11-19T16:16:45,775 DEBUG [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:45,775 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.HStore(327): Store=0dc20a97b5ebf3ba62589849ed17c46f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:16:45,775 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,776 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T16:16:45,776 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dc20a97b5ebf3ba62589849ed17c46f columnFamilyName C 2024-11-19T16:16:45,776 DEBUG [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:45,777 INFO [StoreOpener-0dc20a97b5ebf3ba62589849ed17c46f-1 {}] regionserver.HStore(327): Store=0dc20a97b5ebf3ba62589849ed17c46f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T16:16:45,777 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:45,777 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,778 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,780 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,781 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,781 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,782 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T16:16:45,784 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,785 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 0dc20a97b5ebf3ba62589849ed17c46f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73187589, jitterRate=0.0905800610780716}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T16:16:45,785 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,786 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 0dc20a97b5ebf3ba62589849ed17c46f: Running coprocessor pre-open hook at 1732033005769Writing region info on filesystem at 1732033005769Initializing all the Stores at 1732033005770 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732033005770Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732033005770Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732033005770Cleaning up temporary data from old regions at 1732033005781 (+11 ms)Running coprocessor post-open hooks at 1732033005785 (+4 ms)Region opened successfully at 1732033005786 (+1 ms) 2024-11-19T16:16:45,786 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., pid=145, masterSystemTime=1732033005766 2024-11-19T16:16:45,789 DEBUG [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:45,789 INFO [RS_OPEN_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:45,789 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0dc20a97b5ebf3ba62589849ed17c46f, regionState=OPEN, openSeqNum=5, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:45,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:16:45,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-19T16:16:45,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 in 179 msec 2024-11-19T16:16:45,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-19T16:16:45,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, REOPEN/MOVE in 514 msec 2024-11-19T16:16:45,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-19T16:16:45,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 520 msec 2024-11-19T16:16:45,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-19T16:16:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T16:16:45,805 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:45,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,806 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:45,806 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:45,807 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4faac86d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:45,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:45,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:45,808 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:45,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42d96110, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,810 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:45,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:45,810 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:45,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43246bcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:45,812 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:45,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@662aa739, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:45,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:45,814 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:45,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:45,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:45,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103a2d5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:45,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:45,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:45,815 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:45,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ae3142e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,816 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:45,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:45,817 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6351d54c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:45,818 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@722f0699, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:45,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:45,820 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73538daf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:45,821 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:45,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@223f8e77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:45,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:45,823 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@473bfee0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:45,824 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:45,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef2be01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:45,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:45,826 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c526880, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:45,827 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d6cfa1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:45,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:45,829 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411a41fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:45,830 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19387f76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:45,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:45,831 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a49cd56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:45,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:45,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b9bbcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a8860bb9d6,37355,-1 for getting cluster id 2024-11-19T16:16:45,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T16:16:45,834 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df036147-1459-4aa3-9a3b-991071397ee6' 2024-11-19T16:16:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T16:16:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df036147-1459-4aa3-9a3b-991071397ee6" 2024-11-19T16:16:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201d280b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a8860bb9d6,37355,-1] 2024-11-19T16:16:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T16:16:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:16:45,835 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T16:16:45,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d5a873e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T16:16:45,836 DEBUG [Thread-2435 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:45,836 DEBUG [Thread-2427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:45,836 DEBUG [Thread-2433 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:45,836 DEBUG [Thread-2446 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:45,836 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:45,836 DEBUG [Thread-2429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:45,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:45,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:45,836 DEBUG [Thread-2431 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:45,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:45,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:45,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:45,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:45,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:45,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:45,837 DEBUG [Thread-2442 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-19T16:16:45,837 DEBUG [Thread-2438 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:45,837 DEBUG [Thread-2440 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:45,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:45,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:45,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:45,838 DEBUG [Thread-2440 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:45,838 DEBUG [Thread-2440 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:45,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:45,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:45,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:45,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:45,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:16:45,840 DEBUG [Thread-2444 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T16:16:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T16:16:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:16:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:45,840 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a8860bb9d6,43095,1732032854050, seqNum=-1] 2024-11-19T16:16:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T16:16:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:16:45,841 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:45,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:45,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:45,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T16:16:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:16:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:16:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:16:45,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:16:45,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:16:45,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:16:45,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5] 2024-11-19T16:16:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:45,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:16:45,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:45,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:45,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:45,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:45,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:45,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:45,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e7b8921a94b248f88fcfa670d810f447_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033005846/Put/seqid=0 2024-11-19T16:16:45,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732033005863, max timestamp=1732033005869Num uniques 859; 2024-11-19T16:16:45,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732033005863, max timestamp=1732033005869Num uniques 859; 2024-11-19T16:16:45,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:45,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:45,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732033005863, max timestamp=1732033005869Num uniques 859; 2024-11-19T16:16:45,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:45,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742370_1546 (size=9714) 2024-11-19T16:16:45,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033065881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:45,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033065882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033065883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:45,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033065885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:45,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033065885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T16:16:45,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T16:16:45,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:45,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:45,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:45,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:45,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:45,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033065997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033065997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:45,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033065997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:46,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033065997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:46,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033065998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:46,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T16:16:46,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:46,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:46,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:46,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T16:16:46,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:46,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033066207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033066207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:46,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033066207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033066207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033066207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:46,273 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:46,275 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e7b8921a94b248f88fcfa670d810f447_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e7b8921a94b248f88fcfa670d810f447_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:46,276 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/e7403480156f4915968799af262b7572, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:46,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/e7403480156f4915968799af262b7572 is 175, key is test_row_0/A:col10/1732033005846/Put/seqid=0 2024-11-19T16:16:46,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742371_1547 (size=22361) 2024-11-19T16:16:46,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T16:16:46,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:46,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:46,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:46,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T16:16:46,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:46,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:46,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:46,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T16:16:46,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033066517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:46,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033066517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:46,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033066517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033066517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:46,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033066517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:46,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T16:16:46,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:46,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:46,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:46,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,681 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/e7403480156f4915968799af262b7572 2024-11-19T16:16:46,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/e006825851024fa6857322b8246ae283 is 50, key is test_row_0/B:col10/1732033005846/Put/seqid=0 2024-11-19T16:16:46,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742372_1548 (size=9657) 2024-11-19T16:16:46,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T16:16:46,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:46,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:46,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:46,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T16:16:46,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:46,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:46,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:46,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T16:16:47,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:47,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033067026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033067026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:47,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033067026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:47,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033067026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:47,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033067037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:47,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T16:16:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:47,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:47,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/e006825851024fa6857322b8246ae283 2024-11-19T16:16:47,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/a061afdbdf454a29a3239c824e21959a is 50, key is test_row_0/C:col10/1732033005846/Put/seqid=0 2024-11-19T16:16:47,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742373_1549 (size=9657) 2024-11-19T16:16:47,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/a061afdbdf454a29a3239c824e21959a 2024-11-19T16:16:47,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/e7403480156f4915968799af262b7572 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/e7403480156f4915968799af262b7572 2024-11-19T16:16:47,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/e7403480156f4915968799af262b7572, entries=100, sequenceid=14, filesize=21.8 K 2024-11-19T16:16:47,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/e006825851024fa6857322b8246ae283 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/e006825851024fa6857322b8246ae283 2024-11-19T16:16:47,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/e006825851024fa6857322b8246ae283, entries=100, sequenceid=14, filesize=9.4 K 2024-11-19T16:16:47,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/a061afdbdf454a29a3239c824e21959a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a061afdbdf454a29a3239c824e21959a 2024-11-19T16:16:47,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a061afdbdf454a29a3239c824e21959a, entries=100, sequenceid=14, filesize=9.4 K 2024-11-19T16:16:47,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0dc20a97b5ebf3ba62589849ed17c46f in 1295ms, sequenceid=14, compaction requested=false 2024-11-19T16:16:47,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:47,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T16:16:47,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:47,218 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T16:16:47,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:47,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:47,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:47,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:47,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:47,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:47,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111981b0abe2e85e4729bc0054cb38a304f1_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033005883/Put/seqid=0 2024-11-19T16:16:47,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742374_1550 (size=12154) 2024-11-19T16:16:47,446 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T16:16:47,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:47,630 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111981b0abe2e85e4729bc0054cb38a304f1_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111981b0abe2e85e4729bc0054cb38a304f1_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:47,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0d0c4d61a0e64e35a4b34e841d7e0069, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:47,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0d0c4d61a0e64e35a4b34e841d7e0069 is 175, key is test_row_0/A:col10/1732033005883/Put/seqid=0 2024-11-19T16:16:47,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742375_1551 (size=30955) 2024-11-19T16:16:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T16:16:48,035 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0d0c4d61a0e64e35a4b34e841d7e0069 2024-11-19T16:16:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:48,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:48,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/f9e5baa0e00643fb98253d4ebe1efa9d is 50, key is test_row_0/B:col10/1732033005883/Put/seqid=0 2024-11-19T16:16:48,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742376_1552 (size=12001) 2024-11-19T16:16:48,047 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/f9e5baa0e00643fb98253d4ebe1efa9d 2024-11-19T16:16:48,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033068042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033068045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033068046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033068047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033068048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,053 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 0dc20a97b5ebf3ba62589849ed17c46f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/df2e83dbef4c44ad9f32df6e5d6bba70 is 50, key is test_row_0/C:col10/1732033005883/Put/seqid=0 2024-11-19T16:16:48,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742377_1553 (size=12001) 2024-11-19T16:16:48,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033068156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033068156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033068157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033068157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033068366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033068366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033068366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033068367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,462 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/df2e83dbef4c44ad9f32df6e5d6bba70 2024-11-19T16:16:48,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0d0c4d61a0e64e35a4b34e841d7e0069 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d0c4d61a0e64e35a4b34e841d7e0069 2024-11-19T16:16:48,468 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d0c4d61a0e64e35a4b34e841d7e0069, entries=150, sequenceid=50, filesize=30.2 K 2024-11-19T16:16:48,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/f9e5baa0e00643fb98253d4ebe1efa9d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f9e5baa0e00643fb98253d4ebe1efa9d 2024-11-19T16:16:48,472 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f9e5baa0e00643fb98253d4ebe1efa9d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T16:16:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/df2e83dbef4c44ad9f32df6e5d6bba70 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/df2e83dbef4c44ad9f32df6e5d6bba70 2024-11-19T16:16:48,475 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/df2e83dbef4c44ad9f32df6e5d6bba70, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T16:16:48,476 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 0dc20a97b5ebf3ba62589849ed17c46f in 1258ms, sequenceid=50, compaction requested=false 2024-11-19T16:16:48,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:48,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:48,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-19T16:16:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-19T16:16:48,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-19T16:16:48,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6360 sec 2024-11-19T16:16:48,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.6440 sec 2024-11-19T16:16:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:48,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T16:16:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:48,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111978b51baef2db4ae3a532e0b4df2c00e7_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033008045/Put/seqid=0 2024-11-19T16:16:48,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742378_1554 (size=12154) 2024-11-19T16:16:48,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033068697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732033008694, max timestamp=1732033008701Num uniques 859; 2024-11-19T16:16:48,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:16:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732033008694, max timestamp=1732033008701Num uniques 859; 2024-11-19T16:16:48,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732033008694, max timestamp=1732033008701Num uniques 859; 2024-11-19T16:16:48,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:16:48,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:16:48,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033068730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033068732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033068733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033068807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033068846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033068846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:48,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033068846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:48,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:48,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:49,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033069017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:49,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033069056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:49,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:49,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033069056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:49,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033069056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:49,099 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:49,102 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111978b51baef2db4ae3a532e0b4df2c00e7_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111978b51baef2db4ae3a532e0b4df2c00e7_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:49,102 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/22217a72ff46446f91b5aac4564e0aa7, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:49,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/22217a72ff46446f91b5aac4564e0aa7 is 175, key is test_row_0/A:col10/1732033008045/Put/seqid=0 2024-11-19T16:16:49,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742379_1555 (size=30955) 2024-11-19T16:16:49,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033069327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:49,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033069367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:49,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033069367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033069368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:49,509 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/22217a72ff46446f91b5aac4564e0aa7 2024-11-19T16:16:49,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/8a7ac0b6dda644b8b0ff56a4d17a58bf is 50, key is test_row_0/B:col10/1732033008045/Put/seqid=0 2024-11-19T16:16:49,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742380_1556 (size=12001) 2024-11-19T16:16:49,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033069837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:49,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033069876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:49,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033069876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033069876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:49,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/8a7ac0b6dda644b8b0ff56a4d17a58bf 2024-11-19T16:16:49,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/39235d5096b849099199dfe533d1b8cf is 50, key is test_row_0/C:col10/1732033008045/Put/seqid=0 2024-11-19T16:16:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T16:16:49,997 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:49,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-19T16:16:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T16:16:50,001 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:50,001 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:50,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:50,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742381_1557 (size=12001) 2024-11-19T16:16:50,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/39235d5096b849099199dfe533d1b8cf 2024-11-19T16:16:50,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/22217a72ff46446f91b5aac4564e0aa7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22217a72ff46446f91b5aac4564e0aa7 2024-11-19T16:16:50,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22217a72ff46446f91b5aac4564e0aa7, entries=150, sequenceid=61, filesize=30.2 K 2024-11-19T16:16:50,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/8a7ac0b6dda644b8b0ff56a4d17a58bf as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8a7ac0b6dda644b8b0ff56a4d17a58bf 2024-11-19T16:16:50,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8a7ac0b6dda644b8b0ff56a4d17a58bf, entries=150, sequenceid=61, filesize=11.7 K 2024-11-19T16:16:50,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/39235d5096b849099199dfe533d1b8cf as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/39235d5096b849099199dfe533d1b8cf 2024-11-19T16:16:50,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/39235d5096b849099199dfe533d1b8cf, entries=150, sequenceid=61, filesize=11.7 K 2024-11-19T16:16:50,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0dc20a97b5ebf3ba62589849ed17c46f in 1387ms, sequenceid=61, compaction requested=true 2024-11-19T16:16:50,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:50,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:50,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:50,065 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:50,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:50,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:50,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:50,065 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:50,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:50,066 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84271 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:50,066 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/A is initiating minor compaction (all files) 2024-11-19T16:16:50,066 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/A in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:50,066 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/e7403480156f4915968799af262b7572, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d0c4d61a0e64e35a4b34e841d7e0069, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22217a72ff46446f91b5aac4564e0aa7] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=82.3 K 2024-11-19T16:16:50,066 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:50,066 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/e7403480156f4915968799af262b7572, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d0c4d61a0e64e35a4b34e841d7e0069, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22217a72ff46446f91b5aac4564e0aa7] 2024-11-19T16:16:50,066 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7403480156f4915968799af262b7572, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732033005845 2024-11-19T16:16:50,067 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:50,067 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/B is initiating minor compaction (all files) 2024-11-19T16:16:50,067 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/B in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:50,067 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/e006825851024fa6857322b8246ae283, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f9e5baa0e00643fb98253d4ebe1efa9d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8a7ac0b6dda644b8b0ff56a4d17a58bf] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=32.9 K 2024-11-19T16:16:50,067 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d0c4d61a0e64e35a4b34e841d7e0069, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732033005882 2024-11-19T16:16:50,067 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting e006825851024fa6857322b8246ae283, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732033005845 2024-11-19T16:16:50,067 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22217a72ff46446f91b5aac4564e0aa7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732033008044 2024-11-19T16:16:50,067 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting f9e5baa0e00643fb98253d4ebe1efa9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732033005882 2024-11-19T16:16:50,067 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a7ac0b6dda644b8b0ff56a4d17a58bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732033008044 2024-11-19T16:16:50,079 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:50,081 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111938b65a2c244c4dd99a95129969b15f54_0dc20a97b5ebf3ba62589849ed17c46f store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:50,082 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#B#compaction#478 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:50,082 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/f2c06a63d0054ad5991fe82b81b40844 is 50, key is test_row_0/B:col10/1732033008045/Put/seqid=0 2024-11-19T16:16:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:50,083 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111938b65a2c244c4dd99a95129969b15f54_0dc20a97b5ebf3ba62589849ed17c46f, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:50,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T16:16:50,083 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111938b65a2c244c4dd99a95129969b15f54_0dc20a97b5ebf3ba62589849ed17c46f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:50,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:50,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:50,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:50,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:50,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:50,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:50,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742382_1558 (size=12104) 2024-11-19T16:16:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T16:16:50,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742383_1559 (size=4469) 2024-11-19T16:16:50,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a052d91455024b2f9710703e1d5f394a_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033008732/Put/seqid=0 2024-11-19T16:16:50,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033070139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:50,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742384_1560 (size=12154) 2024-11-19T16:16:50,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T16:16:50,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:50,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:50,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:50,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:50,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033070256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:50,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T16:16:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:50,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:50,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:50,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T16:16:50,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T16:16:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:50,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:50,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033070466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:50,510 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/f2c06a63d0054ad5991fe82b81b40844 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f2c06a63d0054ad5991fe82b81b40844 2024-11-19T16:16:50,514 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/B of 0dc20a97b5ebf3ba62589849ed17c46f into f2c06a63d0054ad5991fe82b81b40844(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:50,514 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:50,514 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/B, priority=13, startTime=1732033010065; duration=0sec 2024-11-19T16:16:50,515 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:50,515 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:B 2024-11-19T16:16:50,515 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:50,516 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:50,516 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/C is initiating minor compaction (all files) 2024-11-19T16:16:50,516 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/C in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:50,516 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a061afdbdf454a29a3239c824e21959a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/df2e83dbef4c44ad9f32df6e5d6bba70, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/39235d5096b849099199dfe533d1b8cf] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=32.9 K 2024-11-19T16:16:50,516 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a061afdbdf454a29a3239c824e21959a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732033005845 2024-11-19T16:16:50,517 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting df2e83dbef4c44ad9f32df6e5d6bba70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732033005882 2024-11-19T16:16:50,517 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 39235d5096b849099199dfe533d1b8cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732033008044 2024-11-19T16:16:50,527 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#A#compaction#477 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:50,527 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/280ef379b0244d6787e204cfb030f41b is 175, key is test_row_0/A:col10/1732033008045/Put/seqid=0 2024-11-19T16:16:50,538 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#C#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:50,538 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/0706846e5eb541d5ae0a276525428446 is 50, key is test_row_0/C:col10/1732033008045/Put/seqid=0 2024-11-19T16:16:50,551 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:50,556 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a052d91455024b2f9710703e1d5f394a_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a052d91455024b2f9710703e1d5f394a_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:50,557 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/2917e8de0d204f2481147cd77cda80e8, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:50,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/2917e8de0d204f2481147cd77cda80e8 is 175, key is test_row_0/A:col10/1732033008732/Put/seqid=0 2024-11-19T16:16:50,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742385_1561 (size=31058) 2024-11-19T16:16:50,578 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/280ef379b0244d6787e204cfb030f41b as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/280ef379b0244d6787e204cfb030f41b 2024-11-19T16:16:50,586 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/A of 0dc20a97b5ebf3ba62589849ed17c46f into 280ef379b0244d6787e204cfb030f41b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:50,586 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:50,586 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/A, priority=13, startTime=1732033010065; duration=0sec 2024-11-19T16:16:50,586 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:50,586 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:A 2024-11-19T16:16:50,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742387_1563 (size=30955) 2024-11-19T16:16:50,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742386_1562 (size=12104) 2024-11-19T16:16:50,616 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/2917e8de0d204f2481147cd77cda80e8 2024-11-19T16:16:50,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T16:16:50,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:50,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:50,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:50,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:50,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:50,623 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/0706846e5eb541d5ae0a276525428446 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/0706846e5eb541d5ae0a276525428446 2024-11-19T16:16:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T16:16:50,630 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/C of 0dc20a97b5ebf3ba62589849ed17c46f into 0706846e5eb541d5ae0a276525428446(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:50,630 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:50,630 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/C, priority=13, startTime=1732033010065; duration=0sec 2024-11-19T16:16:50,630 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:50,631 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:C 2024-11-19T16:16:50,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/34ec5189a9a441828bdfae309b34af9d is 50, key is test_row_0/B:col10/1732033008732/Put/seqid=0 2024-11-19T16:16:50,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742388_1564 (size=12001) 2024-11-19T16:16:50,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/34ec5189a9a441828bdfae309b34af9d 2024-11-19T16:16:50,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/2bb9a5111c63474aa50e15ff23d074e1 is 50, key is test_row_0/C:col10/1732033008732/Put/seqid=0 2024-11-19T16:16:50,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742389_1565 (size=12001) 2024-11-19T16:16:50,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/2bb9a5111c63474aa50e15ff23d074e1 2024-11-19T16:16:50,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/2917e8de0d204f2481147cd77cda80e8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/2917e8de0d204f2481147cd77cda80e8 2024-11-19T16:16:50,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/2917e8de0d204f2481147cd77cda80e8, entries=150, sequenceid=98, filesize=30.2 K 2024-11-19T16:16:50,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/34ec5189a9a441828bdfae309b34af9d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/34ec5189a9a441828bdfae309b34af9d 2024-11-19T16:16:50,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/34ec5189a9a441828bdfae309b34af9d, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T16:16:50,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/2bb9a5111c63474aa50e15ff23d074e1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/2bb9a5111c63474aa50e15ff23d074e1 2024-11-19T16:16:50,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/2bb9a5111c63474aa50e15ff23d074e1, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T16:16:50,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 0dc20a97b5ebf3ba62589849ed17c46f in 663ms, sequenceid=98, compaction requested=false 2024-11-19T16:16:50,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:50,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T16:16:50,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:50,776 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T16:16:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:50,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:50,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f02e507a9be84641a51a12feb2c06795_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033010125/Put/seqid=0 2024-11-19T16:16:50,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742390_1566 (size=12154) 2024-11-19T16:16:50,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732033010871, max timestamp=1732033010882Num uniques 859; 2024-11-19T16:16:50,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:16:50,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732033010871, max timestamp=1732033010882Num uniques 859; 2024-11-19T16:16:50,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:16:50,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732033010871, max timestamp=1732033010882Num uniques 859; 2024-11-19T16:16:50,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:16:50,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033070916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:50,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033070916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033070918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:50,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033070921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:50,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033070922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:50,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:50,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:50,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033071027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033071027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033071027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033071036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033071037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T16:16:51,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:51,241 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f02e507a9be84641a51a12feb2c06795_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f02e507a9be84641a51a12feb2c06795_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:51,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033071237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033071237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033071237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/efc55cda1be0490bb301d3393373b58e, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:51,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/efc55cda1be0490bb301d3393373b58e is 175, key is test_row_0/A:col10/1732033010125/Put/seqid=0 2024-11-19T16:16:51,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742391_1567 (size=30955) 2024-11-19T16:16:51,248 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/efc55cda1be0490bb301d3393373b58e 2024-11-19T16:16:51,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033071247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033071247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/42308408ca974f668accbb3d5ae396ba is 50, key is test_row_0/B:col10/1732033010125/Put/seqid=0 2024-11-19T16:16:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742392_1568 (size=12001) 2024-11-19T16:16:51,270 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/42308408ca974f668accbb3d5ae396ba 2024-11-19T16:16:51,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/1e8abaaa681f4773bec847b652a10202 is 50, key is test_row_0/C:col10/1732033010125/Put/seqid=0 2024-11-19T16:16:51,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742393_1569 (size=12001) 2024-11-19T16:16:51,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033071547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033071547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033071547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033071557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033071566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:51,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:51,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:51,679 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/1e8abaaa681f4773bec847b652a10202 2024-11-19T16:16:51,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/efc55cda1be0490bb301d3393373b58e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/efc55cda1be0490bb301d3393373b58e 2024-11-19T16:16:51,693 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/efc55cda1be0490bb301d3393373b58e, entries=150, sequenceid=111, filesize=30.2 K 2024-11-19T16:16:51,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/42308408ca974f668accbb3d5ae396ba as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/42308408ca974f668accbb3d5ae396ba 2024-11-19T16:16:51,698 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/42308408ca974f668accbb3d5ae396ba, entries=150, sequenceid=111, filesize=11.7 K 2024-11-19T16:16:51,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/1e8abaaa681f4773bec847b652a10202 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/1e8abaaa681f4773bec847b652a10202 2024-11-19T16:16:51,703 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/1e8abaaa681f4773bec847b652a10202, entries=150, sequenceid=111, filesize=11.7 K 2024-11-19T16:16:51,704 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0dc20a97b5ebf3ba62589849ed17c46f in 929ms, sequenceid=111, compaction requested=true 2024-11-19T16:16:51,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:51,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-19T16:16:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-19T16:16:51,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-19T16:16:51,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7050 sec 2024-11-19T16:16:51,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.7130 sec 2024-11-19T16:16:52,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:52,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-19T16:16:52,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:52,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T16:16:52,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:52,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T16:16:52,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:52,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T16:16:52,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c3242a179d594b7a9f8e50b4b9535df8_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033010916/Put/seqid=0 2024-11-19T16:16:52,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033072069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033072070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742394_1570 (size=12304) 2024-11-19T16:16:52,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033072070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033072071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,077 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,080 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c3242a179d594b7a9f8e50b4b9535df8_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c3242a179d594b7a9f8e50b4b9535df8_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:52,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033072077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,081 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/ba252afed0394c7e815c8ce8a88d376d, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:52,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/ba252afed0394c7e815c8ce8a88d376d is 175, key is test_row_0/A:col10/1732033010916/Put/seqid=0 2024-11-19T16:16:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742395_1571 (size=31105) 2024-11-19T16:16:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T16:16:52,147 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:52,147 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-19T16:16:52,148 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T16:16:52,149 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:52,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:52,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033072177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033072187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033072187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033072187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T16:16:52,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T16:16:52,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:52,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:52,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:52,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033072387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033072397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033072397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033072397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T16:16:52,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:52,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:52,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:52,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T16:16:52,486 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/ba252afed0394c7e815c8ce8a88d376d 2024-11-19T16:16:52,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/fc0b389cf71b45e1bcda65a5f828eb19 is 50, key is test_row_0/B:col10/1732033010916/Put/seqid=0 2024-11-19T16:16:52,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742396_1572 (size=12151) 2024-11-19T16:16:52,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/fc0b389cf71b45e1bcda65a5f828eb19 2024-11-19T16:16:52,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/40181433c80e45a1a32f0c6f6736eae5 is 50, key is test_row_0/C:col10/1732033010916/Put/seqid=0 2024-11-19T16:16:52,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742397_1573 (size=12151) 2024-11-19T16:16:52,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T16:16:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:52,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033072697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033072707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033072707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033072707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:52,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T16:16:52,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:52,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:52,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:52,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T16:16:52,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T16:16:52,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:52,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:52,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:52,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:52,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/40181433c80e45a1a32f0c6f6736eae5 2024-11-19T16:16:52,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/ba252afed0394c7e815c8ce8a88d376d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ba252afed0394c7e815c8ce8a88d376d 2024-11-19T16:16:52,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ba252afed0394c7e815c8ce8a88d376d, entries=150, sequenceid=149, filesize=30.4 K 2024-11-19T16:16:52,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/fc0b389cf71b45e1bcda65a5f828eb19 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/fc0b389cf71b45e1bcda65a5f828eb19 2024-11-19T16:16:52,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/fc0b389cf71b45e1bcda65a5f828eb19, entries=150, sequenceid=149, filesize=11.9 K 2024-11-19T16:16:52,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/40181433c80e45a1a32f0c6f6736eae5 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/40181433c80e45a1a32f0c6f6736eae5 2024-11-19T16:16:52,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/40181433c80e45a1a32f0c6f6736eae5, entries=150, sequenceid=149, filesize=11.9 K 2024-11-19T16:16:52,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 0dc20a97b5ebf3ba62589849ed17c46f in 893ms, sequenceid=149, compaction requested=true 2024-11-19T16:16:52,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:52,953 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:52,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:52,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:52,954 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124073 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:52,954 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:52,954 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/A is initiating minor compaction (all files) 2024-11-19T16:16:52,954 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/A in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:52,954 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/280ef379b0244d6787e204cfb030f41b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/2917e8de0d204f2481147cd77cda80e8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/efc55cda1be0490bb301d3393373b58e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ba252afed0394c7e815c8ce8a88d376d] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=121.2 K 2024-11-19T16:16:52,954 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:52,955 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/280ef379b0244d6787e204cfb030f41b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/2917e8de0d204f2481147cd77cda80e8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/efc55cda1be0490bb301d3393373b58e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ba252afed0394c7e815c8ce8a88d376d] 2024-11-19T16:16:52,955 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 280ef379b0244d6787e204cfb030f41b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732033008044 2024-11-19T16:16:52,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:52,955 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:52,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:52,955 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/B is initiating minor compaction (all files) 2024-11-19T16:16:52,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:52,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:52,955 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/B in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:52,955 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f2c06a63d0054ad5991fe82b81b40844, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/34ec5189a9a441828bdfae309b34af9d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/42308408ca974f668accbb3d5ae396ba, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/fc0b389cf71b45e1bcda65a5f828eb19] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=47.1 K 2024-11-19T16:16:52,955 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2917e8de0d204f2481147cd77cda80e8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732033008731 2024-11-19T16:16:52,956 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c06a63d0054ad5991fe82b81b40844, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732033008044 2024-11-19T16:16:52,956 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting efc55cda1be0490bb301d3393373b58e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732033010116 2024-11-19T16:16:52,956 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 34ec5189a9a441828bdfae309b34af9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732033008731 2024-11-19T16:16:52,956 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba252afed0394c7e815c8ce8a88d376d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732033010916 2024-11-19T16:16:52,956 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 42308408ca974f668accbb3d5ae396ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732033010116 2024-11-19T16:16:52,956 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting fc0b389cf71b45e1bcda65a5f828eb19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732033010916 2024-11-19T16:16:52,973 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#B#compaction#489 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:52,974 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/6dbe7fa136254ea88795f4b37fb5fd0d is 50, key is test_row_0/B:col10/1732033010916/Put/seqid=0 2024-11-19T16:16:52,981 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:53,000 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111958a74b94883e41e48df721894552eedd_0dc20a97b5ebf3ba62589849ed17c46f store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:53,003 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111958a74b94883e41e48df721894552eedd_0dc20a97b5ebf3ba62589849ed17c46f, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:53,003 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111958a74b94883e41e48df721894552eedd_0dc20a97b5ebf3ba62589849ed17c46f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:53,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742398_1574 (size=12391) 2024-11-19T16:16:53,035 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/6dbe7fa136254ea88795f4b37fb5fd0d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/6dbe7fa136254ea88795f4b37fb5fd0d 2024-11-19T16:16:53,042 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/B of 0dc20a97b5ebf3ba62589849ed17c46f into 6dbe7fa136254ea88795f4b37fb5fd0d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:53,042 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:53,042 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/B, priority=12, startTime=1732033012954; duration=0sec 2024-11-19T16:16:53,042 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:53,043 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:B 2024-11-19T16:16:53,043 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:53,044 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:53,044 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/C is initiating minor compaction (all files) 2024-11-19T16:16:53,044 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/C in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:53,044 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/0706846e5eb541d5ae0a276525428446, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/2bb9a5111c63474aa50e15ff23d074e1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/1e8abaaa681f4773bec847b652a10202, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/40181433c80e45a1a32f0c6f6736eae5] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=47.1 K 2024-11-19T16:16:53,045 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 0706846e5eb541d5ae0a276525428446, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732033008044 2024-11-19T16:16:53,046 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bb9a5111c63474aa50e15ff23d074e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732033008731 2024-11-19T16:16:53,046 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e8abaaa681f4773bec847b652a10202, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732033010116 2024-11-19T16:16:53,046 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 40181433c80e45a1a32f0c6f6736eae5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732033010916 2024-11-19T16:16:53,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742399_1575 (size=4469) 2024-11-19T16:16:53,050 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#A#compaction#490 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:53,050 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/a0ff963b636d43a8973e77df4edd59d8 is 175, key is test_row_0/A:col10/1732033010916/Put/seqid=0 2024-11-19T16:16:53,063 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#C#compaction#491 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:53,063 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/04cee37b8b0e4c0fb0157901f524146c is 50, key is test_row_0/C:col10/1732033010916/Put/seqid=0 2024-11-19T16:16:53,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T16:16:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:53,066 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T16:16:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:53,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742400_1576 (size=31345) 2024-11-19T16:16:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:53,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:53,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742401_1577 (size=12391) 2024-11-19T16:16:53,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e9abdc8c99254ae68aa3bb1739de2d66_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033012068/Put/seqid=0 2024-11-19T16:16:53,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742402_1578 (size=12304) 2024-11-19T16:16:53,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:53,168 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e9abdc8c99254ae68aa3bb1739de2d66_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e9abdc8c99254ae68aa3bb1739de2d66_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:53,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/68ebaa9a7b2d45db91844d697c8b6f8a, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:53,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/68ebaa9a7b2d45db91844d697c8b6f8a is 175, key is test_row_0/A:col10/1732033012068/Put/seqid=0 2024-11-19T16:16:53,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742403_1579 (size=31105) 2024-11-19T16:16:53,209 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/68ebaa9a7b2d45db91844d697c8b6f8a 2024-11-19T16:16:53,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/bd8ea04c43ed4815a70a5ccb69b46a68 is 50, key is test_row_0/B:col10/1732033012068/Put/seqid=0 2024-11-19T16:16:53,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732033013204, max timestamp=1732033013223Num uniques 859; 2024-11-19T16:16:53,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:53,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732033013204, max timestamp=1732033013223Num uniques 859; 2024-11-19T16:16:53,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:16:53,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732033013204, max timestamp=1732033013223Num uniques 859; 2024-11-19T16:16:53,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T16:16:53,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742404_1580 (size=12151) 2024-11-19T16:16:53,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T16:16:53,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732033013277, max timestamp=1732033013291Num uniques 1068; 2024-11-19T16:16:53,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732033013277, max timestamp=1732033013291Num uniques 1018; 2024-11-19T16:16:53,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732033013283, max timestamp=1732033013291Num uniques 1018; 2024-11-19T16:16:53,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-19T16:16:53,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-19T16:16:53,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-19T16:16:53,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033073347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033073349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033073350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033073352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033073358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033073466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033073466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033073466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033073466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033073477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,488 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/a0ff963b636d43a8973e77df4edd59d8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/a0ff963b636d43a8973e77df4edd59d8 2024-11-19T16:16:53,491 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/A of 0dc20a97b5ebf3ba62589849ed17c46f into a0ff963b636d43a8973e77df4edd59d8(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:53,491 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:53,491 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/A, priority=12, startTime=1732033012953; duration=0sec 2024-11-19T16:16:53,491 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:53,491 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:A 2024-11-19T16:16:53,541 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/04cee37b8b0e4c0fb0157901f524146c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/04cee37b8b0e4c0fb0157901f524146c 2024-11-19T16:16:53,544 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/C of 0dc20a97b5ebf3ba62589849ed17c46f into 04cee37b8b0e4c0fb0157901f524146c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:53,544 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:53,544 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/C, priority=12, startTime=1732033012955; duration=0sec 2024-11-19T16:16:53,545 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:53,545 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:C 2024-11-19T16:16:53,598 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T16:16:53,598 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T16:16:53,642 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/bd8ea04c43ed4815a70a5ccb69b46a68 2024-11-19T16:16:53,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/234d87c8298440249376c0522fab7983 is 50, key is test_row_0/C:col10/1732033012068/Put/seqid=0 2024-11-19T16:16:53,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742405_1581 (size=12151) 2024-11-19T16:16:53,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033073676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033073676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033073676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033073677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033073687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033073987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033073987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033073987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033073988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:53,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033073997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,054 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/234d87c8298440249376c0522fab7983 2024-11-19T16:16:54,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/68ebaa9a7b2d45db91844d697c8b6f8a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/68ebaa9a7b2d45db91844d697c8b6f8a 2024-11-19T16:16:54,063 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/68ebaa9a7b2d45db91844d697c8b6f8a, entries=150, sequenceid=159, filesize=30.4 K 2024-11-19T16:16:54,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/bd8ea04c43ed4815a70a5ccb69b46a68 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/bd8ea04c43ed4815a70a5ccb69b46a68 2024-11-19T16:16:54,068 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/bd8ea04c43ed4815a70a5ccb69b46a68, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T16:16:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/234d87c8298440249376c0522fab7983 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/234d87c8298440249376c0522fab7983 2024-11-19T16:16:54,077 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/234d87c8298440249376c0522fab7983, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T16:16:54,079 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=142.33 KB/145742 for 0dc20a97b5ebf3ba62589849ed17c46f in 1013ms, sequenceid=159, compaction requested=false 2024-11-19T16:16:54,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:54,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:54,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-19T16:16:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-19T16:16:54,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-19T16:16:54,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9320 sec 2024-11-19T16:16:54,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.9380 sec 2024-11-19T16:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T16:16:54,308 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:54,308 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-19T16:16:54,310 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T16:16:54,310 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:54,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T16:16:54,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T16:16:54,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:54,467 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=142.33 KB heapSize=402.63 KB 2024-11-19T16:16:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:54,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199454a12322c848a8943658a3e40f3636_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033013333/Put/seqid=0 2024-11-19T16:16:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:54,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:54,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742406_1582 (size=12304) 2024-11-19T16:16:54,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:54,507 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199454a12322c848a8943658a3e40f3636_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199454a12322c848a8943658a3e40f3636_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:54,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0a4b0d72187d4ed3b030cbb1540a7274, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:54,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0a4b0d72187d4ed3b030cbb1540a7274 is 175, key is test_row_0/A:col10/1732033013333/Put/seqid=0 2024-11-19T16:16:54,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:54,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033074505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033074505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033074507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742407_1583 (size=31105) 2024-11-19T16:16:54,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033074514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:54,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033074514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:54,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,525 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0a4b0d72187d4ed3b030cbb1540a7274 2024-11-19T16:16:54,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/1de0f3d8033943b3a104348c330e1ebf is 50, key is test_row_0/B:col10/1732033013333/Put/seqid=0 2024-11-19T16:16:54,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742408_1584 (size=12151) 2024-11-19T16:16:54,544 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/1de0f3d8033943b3a104348c330e1ebf 2024-11-19T16:16:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/c55c8ecfd2fc44e39170f49c380ebd69 is 50, key is test_row_0/C:col10/1732033013333/Put/seqid=0 2024-11-19T16:16:54,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742409_1585 (size=12151) 2024-11-19T16:16:54,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033074617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033074617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T16:16:54,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033074627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:54,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033074627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033074826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:54,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033074826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033074836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033074836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T16:16:54,957 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.93 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/c55c8ecfd2fc44e39170f49c380ebd69 2024-11-19T16:16:54,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0a4b0d72187d4ed3b030cbb1540a7274 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0a4b0d72187d4ed3b030cbb1540a7274 2024-11-19T16:16:54,966 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0a4b0d72187d4ed3b030cbb1540a7274, entries=150, sequenceid=215, filesize=30.4 K 2024-11-19T16:16:54,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/1de0f3d8033943b3a104348c330e1ebf as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/1de0f3d8033943b3a104348c330e1ebf 2024-11-19T16:16:54,971 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/1de0f3d8033943b3a104348c330e1ebf, entries=150, sequenceid=215, filesize=11.9 K 2024-11-19T16:16:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/c55c8ecfd2fc44e39170f49c380ebd69 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/c55c8ecfd2fc44e39170f49c380ebd69 2024-11-19T16:16:54,976 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/c55c8ecfd2fc44e39170f49c380ebd69, entries=150, sequenceid=215, filesize=11.9 K 2024-11-19T16:16:54,977 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~142.33 KB/145742, heapSize ~402.58 KB/412240, currentSize=46.96 KB/48090 for 0dc20a97b5ebf3ba62589849ed17c46f in 510ms, sequenceid=215, compaction requested=true 2024-11-19T16:16:54,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:54,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:54,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-19T16:16:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-19T16:16:54,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-19T16:16:54,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 667 msec 2024-11-19T16:16:54,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 672 msec 2024-11-19T16:16:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:55,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T16:16:55,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:55,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:55,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:55,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:55,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:55,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:55,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191bd183ef408d4136afa0cf19afdad502_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033015138/Put/seqid=0 2024-11-19T16:16:55,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742410_1586 (size=12304) 2024-11-19T16:16:55,148 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:55,152 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191bd183ef408d4136afa0cf19afdad502_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191bd183ef408d4136afa0cf19afdad502_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:55,153 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/ea917fef73e847f2b26fd1a27ef0e9c1, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:55,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/ea917fef73e847f2b26fd1a27ef0e9c1 is 175, key is test_row_0/A:col10/1732033015138/Put/seqid=0 2024-11-19T16:16:55,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742411_1587 (size=31105) 2024-11-19T16:16:55,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033075169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033075171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033075172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732033015163, max timestamp=1732033015174Num uniques 859; 2024-11-19T16:16:55,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732033015163, max timestamp=1732033015174Num uniques 859; 2024-11-19T16:16:55,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:55,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:55,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732033015163, max timestamp=1732033015174Num uniques 859; 2024-11-19T16:16:55,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32a8860bb9d6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T16:16:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033075261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033075286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033075286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033075286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033075376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T16:16:55,447 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:55,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-19T16:16:55,450 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:55,451 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:55,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T16:16:55,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033075497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033075497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033075497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033075527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,532 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 0dc20a97b5ebf3ba62589849ed17c46f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T16:16:55,564 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/ea917fef73e847f2b26fd1a27ef0e9c1 2024-11-19T16:16:55,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/da470c302b334971b489ffbccfda5ca8 is 50, key is test_row_0/B:col10/1732033015138/Put/seqid=0 2024-11-19T16:16:55,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742412_1588 (size=12151) 2024-11-19T16:16:55,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/da470c302b334971b489ffbccfda5ca8 2024-11-19T16:16:55,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/7356de6343ba4eb791e5767da7730e19 is 50, key is test_row_0/C:col10/1732033015138/Put/seqid=0 2024-11-19T16:16:55,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742413_1589 (size=12151) 2024-11-19T16:16:55,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/7356de6343ba4eb791e5767da7730e19 2024-11-19T16:16:55,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/ea917fef73e847f2b26fd1a27ef0e9c1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ea917fef73e847f2b26fd1a27ef0e9c1 2024-11-19T16:16:55,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ea917fef73e847f2b26fd1a27ef0e9c1, entries=150, sequenceid=226, filesize=30.4 K 2024-11-19T16:16:55,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/da470c302b334971b489ffbccfda5ca8 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/da470c302b334971b489ffbccfda5ca8 2024-11-19T16:16:55,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/da470c302b334971b489ffbccfda5ca8, entries=150, sequenceid=226, filesize=11.9 K 2024-11-19T16:16:55,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/7356de6343ba4eb791e5767da7730e19 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/7356de6343ba4eb791e5767da7730e19 2024-11-19T16:16:55,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033075586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/7356de6343ba4eb791e5767da7730e19, entries=150, sequenceid=226, filesize=11.9 K 2024-11-19T16:16:55,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0dc20a97b5ebf3ba62589849ed17c46f in 453ms, sequenceid=226, compaction requested=true 2024-11-19T16:16:55,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:55,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:55,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:55,592 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:55,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:55,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:55,592 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:55,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:55,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:55,593 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:55,593 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:55,593 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/A is initiating minor compaction (all files) 2024-11-19T16:16:55,593 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/B is initiating minor compaction (all files) 2024-11-19T16:16:55,593 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/A in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:55,593 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/B in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:55,593 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/a0ff963b636d43a8973e77df4edd59d8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/68ebaa9a7b2d45db91844d697c8b6f8a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0a4b0d72187d4ed3b030cbb1540a7274, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ea917fef73e847f2b26fd1a27ef0e9c1] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=121.7 K 2024-11-19T16:16:55,593 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/6dbe7fa136254ea88795f4b37fb5fd0d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/bd8ea04c43ed4815a70a5ccb69b46a68, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/1de0f3d8033943b3a104348c330e1ebf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/da470c302b334971b489ffbccfda5ca8] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=47.7 K 2024-11-19T16:16:55,593 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:55,593 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/a0ff963b636d43a8973e77df4edd59d8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/68ebaa9a7b2d45db91844d697c8b6f8a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0a4b0d72187d4ed3b030cbb1540a7274, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ea917fef73e847f2b26fd1a27ef0e9c1] 2024-11-19T16:16:55,594 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0ff963b636d43a8973e77df4edd59d8, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732033010916 2024-11-19T16:16:55,594 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dbe7fa136254ea88795f4b37fb5fd0d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732033010916 2024-11-19T16:16:55,594 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68ebaa9a7b2d45db91844d697c8b6f8a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732033012068 2024-11-19T16:16:55,594 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting bd8ea04c43ed4815a70a5ccb69b46a68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732033012068 2024-11-19T16:16:55,594 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a4b0d72187d4ed3b030cbb1540a7274, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732033013333 2024-11-19T16:16:55,594 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 1de0f3d8033943b3a104348c330e1ebf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732033013333 2024-11-19T16:16:55,594 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea917fef73e847f2b26fd1a27ef0e9c1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732033014501 2024-11-19T16:16:55,594 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting da470c302b334971b489ffbccfda5ca8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732033014501 2024-11-19T16:16:55,599 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:55,600 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119cc5ab6552fd64580a7a7d3a3a1ad2f40_0dc20a97b5ebf3ba62589849ed17c46f store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:55,601 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#B#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:55,602 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/312f21547fd148ccb2f2416d464d7856 is 50, key is test_row_0/B:col10/1732033015138/Put/seqid=0 2024-11-19T16:16:55,603 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119cc5ab6552fd64580a7a7d3a3a1ad2f40_0dc20a97b5ebf3ba62589849ed17c46f, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:55,603 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cc5ab6552fd64580a7a7d3a3a1ad2f40_0dc20a97b5ebf3ba62589849ed17c46f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:55,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T16:16:55,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:55,604 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T16:16:55,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:55,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:55,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:55,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:55,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:55,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:16:55,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742414_1590 (size=12527) 2024-11-19T16:16:55,613 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/312f21547fd148ccb2f2416d464d7856 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/312f21547fd148ccb2f2416d464d7856 2024-11-19T16:16:55,617 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/B of 0dc20a97b5ebf3ba62589849ed17c46f into 312f21547fd148ccb2f2416d464d7856(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:55,617 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:55,617 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/B, priority=12, startTime=1732033015592; duration=0sec 2024-11-19T16:16:55,617 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:55,617 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:B 2024-11-19T16:16:55,618 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:16:55,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742415_1591 (size=4469) 2024-11-19T16:16:55,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199da4f1c984274e95a3e22d831783624a_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033015246/Put/seqid=0 2024-11-19T16:16:55,621 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#A#compaction#502 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:55,621 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/11567aeacaf242c7960cf8182023fc42 is 175, key is test_row_0/A:col10/1732033015138/Put/seqid=0 2024-11-19T16:16:55,621 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:16:55,621 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/C is initiating minor compaction (all files) 2024-11-19T16:16:55,621 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/C in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:55,621 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/04cee37b8b0e4c0fb0157901f524146c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/234d87c8298440249376c0522fab7983, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/c55c8ecfd2fc44e39170f49c380ebd69, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/7356de6343ba4eb791e5767da7730e19] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=47.7 K 2024-11-19T16:16:55,622 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 04cee37b8b0e4c0fb0157901f524146c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732033010916 2024-11-19T16:16:55,622 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 234d87c8298440249376c0522fab7983, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732033012068 2024-11-19T16:16:55,622 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting c55c8ecfd2fc44e39170f49c380ebd69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732033013333 2024-11-19T16:16:55,622 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 7356de6343ba4eb791e5767da7730e19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732033014501 2024-11-19T16:16:55,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742417_1593 (size=31481) 2024-11-19T16:16:55,627 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/11567aeacaf242c7960cf8182023fc42 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/11567aeacaf242c7960cf8182023fc42 2024-11-19T16:16:55,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742416_1592 (size=12454) 2024-11-19T16:16:55,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:55,631 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/A of 0dc20a97b5ebf3ba62589849ed17c46f into 11567aeacaf242c7960cf8182023fc42(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:55,631 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:55,631 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/A, priority=12, startTime=1732033015592; duration=0sec 2024-11-19T16:16:55,631 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:55,631 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:A 2024-11-19T16:16:55,634 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#C#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:55,634 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/921cbfb66853424890bdea8ae72177f3 is 50, key is test_row_0/C:col10/1732033015138/Put/seqid=0 2024-11-19T16:16:55,635 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199da4f1c984274e95a3e22d831783624a_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199da4f1c984274e95a3e22d831783624a_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:55,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/96098c87254940fb8c53f09ad54d9cfe, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:55,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/96098c87254940fb8c53f09ad54d9cfe is 175, key is test_row_0/A:col10/1732033015246/Put/seqid=0 2024-11-19T16:16:55,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742418_1594 (size=12527) 2024-11-19T16:16:55,643 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/921cbfb66853424890bdea8ae72177f3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/921cbfb66853424890bdea8ae72177f3 2024-11-19T16:16:55,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742419_1595 (size=31255) 2024-11-19T16:16:55,646 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/96098c87254940fb8c53f09ad54d9cfe 2024-11-19T16:16:55,646 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/C of 0dc20a97b5ebf3ba62589849ed17c46f into 921cbfb66853424890bdea8ae72177f3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:55,646 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:55,646 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/C, priority=12, startTime=1732033015592; duration=0sec 2024-11-19T16:16:55,647 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:55,647 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:C 2024-11-19T16:16:55,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/e970d51f8da64a3e9a0276f53d025748 is 50, key is test_row_0/B:col10/1732033015246/Put/seqid=0 2024-11-19T16:16:55,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742420_1596 (size=12301) 2024-11-19T16:16:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T16:16:55,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:55,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033075821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033075822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033075823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033075897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033075937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033075937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033075937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,057 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/e970d51f8da64a3e9a0276f53d025748 2024-11-19T16:16:56,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/39c03690f7a84f8da68081eaa99a23e3 is 50, key is test_row_0/C:col10/1732033015246/Put/seqid=0 2024-11-19T16:16:56,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742421_1597 (size=12301) 2024-11-19T16:16:56,075 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/39c03690f7a84f8da68081eaa99a23e3 2024-11-19T16:16:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T16:16:56,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/96098c87254940fb8c53f09ad54d9cfe as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/96098c87254940fb8c53f09ad54d9cfe 2024-11-19T16:16:56,085 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/96098c87254940fb8c53f09ad54d9cfe, entries=150, sequenceid=261, filesize=30.5 K 2024-11-19T16:16:56,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/e970d51f8da64a3e9a0276f53d025748 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/e970d51f8da64a3e9a0276f53d025748 2024-11-19T16:16:56,089 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/e970d51f8da64a3e9a0276f53d025748, entries=150, sequenceid=261, filesize=12.0 K 2024-11-19T16:16:56,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/39c03690f7a84f8da68081eaa99a23e3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/39c03690f7a84f8da68081eaa99a23e3 2024-11-19T16:16:56,095 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/39c03690f7a84f8da68081eaa99a23e3, entries=150, sequenceid=261, filesize=12.0 K 2024-11-19T16:16:56,096 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 0dc20a97b5ebf3ba62589849ed17c46f in 492ms, sequenceid=261, compaction requested=false 2024-11-19T16:16:56,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:56,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:56,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-19T16:16:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-19T16:16:56,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-19T16:16:56,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 646 msec 2024-11-19T16:16:56,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 651 msec 2024-11-19T16:16:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:56,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T16:16:56,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:56,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:56,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:56,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:56,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:56,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:56,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b22234e3d16f4a0ba9d14f0658160c63_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033015815/Put/seqid=0 2024-11-19T16:16:56,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742422_1598 (size=12454) 2024-11-19T16:16:56,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033076173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033076176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033076178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033076287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:56,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:56,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033076287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033076287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033076407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033076496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:56,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033076496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033076496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,557 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:56,560 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b22234e3d16f4a0ba9d14f0658160c63_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b22234e3d16f4a0ba9d14f0658160c63_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:56,561 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/715a2f4afbbb4021a67f8a1dec8a93af, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:56,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/715a2f4afbbb4021a67f8a1dec8a93af is 175, key is test_row_0/A:col10/1732033015815/Put/seqid=0 2024-11-19T16:16:56,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742423_1599 (size=31255) 2024-11-19T16:16:56,566 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/715a2f4afbbb4021a67f8a1dec8a93af 2024-11-19T16:16:56,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/78c03585884646b8a076c93ebe6f088d is 50, key is test_row_0/B:col10/1732033015815/Put/seqid=0 2024-11-19T16:16:56,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742424_1600 (size=12301) 2024-11-19T16:16:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T16:16:56,587 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:16:56,588 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:16:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-19T16:16:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T16:16:56,590 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:16:56,590 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:16:56,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:16:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T16:16:56,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T16:16:56,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:56,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:56,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:56,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:56,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:56,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:56,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033076807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033076807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:56,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033076807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:56,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:56,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T16:16:56,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:56,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:56,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:56,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:56,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T16:16:56,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/78c03585884646b8a076c93ebe6f088d 2024-11-19T16:16:56,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/0d4a4e109d674ae5acd4c75a63808204 is 50, key is test_row_0/C:col10/1732033015815/Put/seqid=0 2024-11-19T16:16:57,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742425_1601 (size=12301) 2024-11-19T16:16:57,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T16:16:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:57,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T16:16:57,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:57,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T16:16:57,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033077316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:57,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033077316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:57,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033077316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:57,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:57,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T16:16:57,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:57,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/0d4a4e109d674ae5acd4c75a63808204 2024-11-19T16:16:57,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/715a2f4afbbb4021a67f8a1dec8a93af as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/715a2f4afbbb4021a67f8a1dec8a93af 2024-11-19T16:16:57,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/715a2f4afbbb4021a67f8a1dec8a93af, entries=150, sequenceid=278, filesize=30.5 K 2024-11-19T16:16:57,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/78c03585884646b8a076c93ebe6f088d as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/78c03585884646b8a076c93ebe6f088d 2024-11-19T16:16:57,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/78c03585884646b8a076c93ebe6f088d, entries=150, sequenceid=278, filesize=12.0 K 2024-11-19T16:16:57,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/0d4a4e109d674ae5acd4c75a63808204 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/0d4a4e109d674ae5acd4c75a63808204 2024-11-19T16:16:57,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/0d4a4e109d674ae5acd4c75a63808204, entries=150, sequenceid=278, filesize=12.0 K 2024-11-19T16:16:57,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0dc20a97b5ebf3ba62589849ed17c46f in 1274ms, sequenceid=278, compaction requested=true 2024-11-19T16:16:57,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:57,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:16:57,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:57,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:16:57,422 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:57,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:57,422 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:57,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:16:57,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:57,423 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:57,423 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/B is initiating minor compaction (all files) 2024-11-19T16:16:57,423 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/B in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,423 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/312f21547fd148ccb2f2416d464d7856, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/e970d51f8da64a3e9a0276f53d025748, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/78c03585884646b8a076c93ebe6f088d] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=36.3 K 2024-11-19T16:16:57,423 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:57,423 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/A is initiating minor compaction (all files) 2024-11-19T16:16:57,424 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/A in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,424 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/11567aeacaf242c7960cf8182023fc42, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/96098c87254940fb8c53f09ad54d9cfe, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/715a2f4afbbb4021a67f8a1dec8a93af] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=91.8 K 2024-11-19T16:16:57,424 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,424 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/11567aeacaf242c7960cf8182023fc42, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/96098c87254940fb8c53f09ad54d9cfe, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/715a2f4afbbb4021a67f8a1dec8a93af] 2024-11-19T16:16:57,424 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 312f21547fd148ccb2f2416d464d7856, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732033014501 2024-11-19T16:16:57,424 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11567aeacaf242c7960cf8182023fc42, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732033014501 2024-11-19T16:16:57,424 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting e970d51f8da64a3e9a0276f53d025748, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732033015246 2024-11-19T16:16:57,424 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96098c87254940fb8c53f09ad54d9cfe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732033015246 2024-11-19T16:16:57,424 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 78c03585884646b8a076c93ebe6f088d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732033015815 2024-11-19T16:16:57,425 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 715a2f4afbbb4021a67f8a1dec8a93af, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732033015815 2024-11-19T16:16:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:57,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T16:16:57,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:57,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:57,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:57,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:57,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:57,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:57,431 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:57,432 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#B#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:57,432 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/5c619bec28cb4cf0aa15098f8f99fbcc is 50, key is test_row_0/B:col10/1732033015815/Put/seqid=0 2024-11-19T16:16:57,433 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111952c3ed94f27d420ba6db24e65014f264_0dc20a97b5ebf3ba62589849ed17c46f store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:57,434 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111952c3ed94f27d420ba6db24e65014f264_0dc20a97b5ebf3ba62589849ed17c46f, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:57,435 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111952c3ed94f27d420ba6db24e65014f264_0dc20a97b5ebf3ba62589849ed17c46f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:57,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119904900ee1f0a48c3b7ccc6490e1946b9_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033016172/Put/seqid=0 2024-11-19T16:16:57,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742427_1603 (size=12779) 2024-11-19T16:16:57,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742426_1602 (size=4469) 2024-11-19T16:16:57,456 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/5c619bec28cb4cf0aa15098f8f99fbcc as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5c619bec28cb4cf0aa15098f8f99fbcc 2024-11-19T16:16:57,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742428_1604 (size=12454) 2024-11-19T16:16:57,470 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/B of 0dc20a97b5ebf3ba62589849ed17c46f into 5c619bec28cb4cf0aa15098f8f99fbcc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:57,470 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:57,470 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/B, priority=13, startTime=1732033017422; duration=0sec 2024-11-19T16:16:57,470 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:16:57,470 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:B 2024-11-19T16:16:57,470 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:16:57,471 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:16:57,471 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/C is initiating minor compaction (all files) 2024-11-19T16:16:57,471 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/C in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,471 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/921cbfb66853424890bdea8ae72177f3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/39c03690f7a84f8da68081eaa99a23e3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/0d4a4e109d674ae5acd4c75a63808204] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=36.3 K 2024-11-19T16:16:57,472 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 921cbfb66853424890bdea8ae72177f3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732033014501 2024-11-19T16:16:57,472 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 39c03690f7a84f8da68081eaa99a23e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732033015246 2024-11-19T16:16:57,472 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d4a4e109d674ae5acd4c75a63808204, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732033015815 2024-11-19T16:16:57,476 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#C#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:57,477 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/8d671518c60d47f4911bb28afebbf5c3 is 50, key is test_row_0/C:col10/1732033015815/Put/seqid=0 2024-11-19T16:16:57,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742429_1605 (size=12779) 2024-11-19T16:16:57,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T16:16:57,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:57,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033077503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:57,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033077556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:57,559 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 0dc20a97b5ebf3ba62589849ed17c46f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:57,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033077616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:57,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T16:16:57,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:57,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T16:16:57,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T16:16:57,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:57,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:16:57,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033077827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:57,857 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#A#compaction#511 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:16:57,857 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/87317cbc64c04c16abf7114410d29039 is 175, key is test_row_0/A:col10/1732033015815/Put/seqid=0 2024-11-19T16:16:57,858 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:57,861 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119904900ee1f0a48c3b7ccc6490e1946b9_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119904900ee1f0a48c3b7ccc6490e1946b9_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:57,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742430_1606 (size=31733) 2024-11-19T16:16:57,862 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/ffb945437dea40849b142e149275cb96, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:57,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/ffb945437dea40849b142e149275cb96 is 175, key is test_row_0/A:col10/1732033016172/Put/seqid=0 2024-11-19T16:16:57,866 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/87317cbc64c04c16abf7114410d29039 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/87317cbc64c04c16abf7114410d29039 2024-11-19T16:16:57,869 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/A of 0dc20a97b5ebf3ba62589849ed17c46f into 87317cbc64c04c16abf7114410d29039(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:57,869 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:57,869 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/A, priority=13, startTime=1732033017422; duration=0sec 2024-11-19T16:16:57,869 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:57,869 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:A 2024-11-19T16:16:57,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742431_1607 (size=31255) 2024-11-19T16:16:57,876 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/ffb945437dea40849b142e149275cb96 2024-11-19T16:16:57,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/82189379da834333b255dc45a9e604b6 is 50, key is test_row_0/B:col10/1732033016172/Put/seqid=0 2024-11-19T16:16:57,894 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/8d671518c60d47f4911bb28afebbf5c3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/8d671518c60d47f4911bb28afebbf5c3 2024-11-19T16:16:57,899 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/C of 0dc20a97b5ebf3ba62589849ed17c46f into 8d671518c60d47f4911bb28afebbf5c3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:16:57,899 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:57,899 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/C, priority=13, startTime=1732033017422; duration=0sec 2024-11-19T16:16:57,899 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:16:57,899 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:C 2024-11-19T16:16:57,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742432_1608 (size=12301) 2024-11-19T16:16:57,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/82189379da834333b255dc45a9e604b6 2024-11-19T16:16:57,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/a408e7591c4e459d8eefcd47b920ad34 is 50, key is test_row_0/C:col10/1732033016172/Put/seqid=0 2024-11-19T16:16:57,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742433_1609 (size=12301) 2024-11-19T16:16:57,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/a408e7591c4e459d8eefcd47b920ad34 2024-11-19T16:16:57,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/ffb945437dea40849b142e149275cb96 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ffb945437dea40849b142e149275cb96 2024-11-19T16:16:57,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ffb945437dea40849b142e149275cb96, entries=150, sequenceid=297, filesize=30.5 K 2024-11-19T16:16:57,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/82189379da834333b255dc45a9e604b6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/82189379da834333b255dc45a9e604b6 2024-11-19T16:16:57,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/82189379da834333b255dc45a9e604b6, entries=150, sequenceid=297, filesize=12.0 K 2024-11-19T16:16:57,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/a408e7591c4e459d8eefcd47b920ad34 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a408e7591c4e459d8eefcd47b920ad34 2024-11-19T16:16:57,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a408e7591c4e459d8eefcd47b920ad34, entries=150, sequenceid=297, filesize=12.0 K 2024-11-19T16:16:57,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0dc20a97b5ebf3ba62589849ed17c46f in 501ms, sequenceid=297, compaction requested=false 2024-11-19T16:16:57,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:57,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T16:16:57,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:57,964 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T16:16:57,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:57,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:57,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:57,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:57,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:57,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:57,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111965b21447b475488895935b865bae42be_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033017501/Put/seqid=0 2024-11-19T16:16:57,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742434_1610 (size=12454) 2024-11-19T16:16:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:58,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:16:58,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033078208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:58,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033078317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:58,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033078327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:58,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033078328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:58,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:58,331 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 0dc20a97b5ebf3ba62589849ed17c46f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033078328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:58,331 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 0dc20a97b5ebf3ba62589849ed17c46f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:58,332 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_0' in 0dc20a97b5ebf3ba62589849ed17c46f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:58,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:58,376 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111965b21447b475488895935b865bae42be_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111965b21447b475488895935b865bae42be_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:58,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/22eed6384e5c4e419ee2e549dfc87bac, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:58,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/22eed6384e5c4e419ee2e549dfc87bac is 175, key is test_row_0/A:col10/1732033017501/Put/seqid=0 2024-11-19T16:16:58,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742435_1611 (size=31255) 2024-11-19T16:16:58,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033078526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T16:16:58,790 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/22eed6384e5c4e419ee2e549dfc87bac 2024-11-19T16:16:58,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/23bfe436a74b4329af5920a3851770cb is 50, key is test_row_0/B:col10/1732033017501/Put/seqid=0 2024-11-19T16:16:58,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742436_1612 (size=12301) 2024-11-19T16:16:58,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033078837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:59,202 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/23bfe436a74b4329af5920a3851770cb 2024-11-19T16:16:59,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/ef1ef1cc0c1647e3bb041ce4ce2f80db is 50, key is test_row_0/C:col10/1732033017501/Put/seqid=0 2024-11-19T16:16:59,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742437_1613 (size=12301) 2024-11-19T16:16:59,210 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/ef1ef1cc0c1647e3bb041ce4ce2f80db 2024-11-19T16:16:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/22eed6384e5c4e419ee2e549dfc87bac as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22eed6384e5c4e419ee2e549dfc87bac 2024-11-19T16:16:59,215 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22eed6384e5c4e419ee2e549dfc87bac, entries=150, sequenceid=313, filesize=30.5 K 2024-11-19T16:16:59,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/23bfe436a74b4329af5920a3851770cb as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/23bfe436a74b4329af5920a3851770cb 2024-11-19T16:16:59,219 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/23bfe436a74b4329af5920a3851770cb, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T16:16:59,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/ef1ef1cc0c1647e3bb041ce4ce2f80db as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/ef1ef1cc0c1647e3bb041ce4ce2f80db 2024-11-19T16:16:59,222 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/ef1ef1cc0c1647e3bb041ce4ce2f80db, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T16:16:59,223 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0dc20a97b5ebf3ba62589849ed17c46f in 1258ms, sequenceid=313, compaction requested=true 2024-11-19T16:16:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:16:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:16:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-19T16:16:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-19T16:16:59,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-19T16:16:59,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6340 sec 2024-11-19T16:16:59,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.6380 sec 2024-11-19T16:16:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:59,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T16:16:59,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:16:59,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:59,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:16:59,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:59,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:16:59,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:16:59,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c79cc536ee7e40f085428b7472fd8da5_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033019349/Put/seqid=0 2024-11-19T16:16:59,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742438_1614 (size=14994) 2024-11-19T16:16:59,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033079397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:59,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033079507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:59,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:16:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033079717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:16:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:16:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:16:59,766 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:16:59,769 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c79cc536ee7e40f085428b7472fd8da5_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c79cc536ee7e40f085428b7472fd8da5_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:16:59,769 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/d7b84eb8b287488db151f75ff9ad3abc, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:16:59,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/d7b84eb8b287488db151f75ff9ad3abc is 175, key is test_row_0/A:col10/1732033019349/Put/seqid=0 2024-11-19T16:16:59,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742439_1615 (size=39949) 2024-11-19T16:17:00,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033080027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:00,174 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/d7b84eb8b287488db151f75ff9ad3abc 2024-11-19T16:17:00,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/8d02dbe582cd4579b611e91d6471c51a is 50, key is test_row_0/B:col10/1732033019349/Put/seqid=0 2024-11-19T16:17:00,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742440_1616 (size=12301) 2024-11-19T16:17:00,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033080337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:00,339 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 0dc20a97b5ebf3ba62589849ed17c46f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:17:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:00,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033080347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:00,348 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_1' in 0dc20a97b5ebf3ba62589849ed17c46f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:17:00,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:00,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:00,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:00,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033080356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:00,360 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_0' in 0dc20a97b5ebf3ba62589849ed17c46f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:17:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:00,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033080536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:00,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/8d02dbe582cd4579b611e91d6471c51a 2024-11-19T16:17:00,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/a904de6e0f2842bca9d148f2aca012d6 is 50, key is test_row_0/C:col10/1732033019349/Put/seqid=0 2024-11-19T16:17:00,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742441_1617 (size=12301) 2024-11-19T16:17:00,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/a904de6e0f2842bca9d148f2aca012d6 2024-11-19T16:17:00,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/d7b84eb8b287488db151f75ff9ad3abc as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/d7b84eb8b287488db151f75ff9ad3abc 2024-11-19T16:17:00,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/d7b84eb8b287488db151f75ff9ad3abc, entries=200, sequenceid=333, filesize=39.0 K 2024-11-19T16:17:00,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/8d02dbe582cd4579b611e91d6471c51a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8d02dbe582cd4579b611e91d6471c51a 2024-11-19T16:17:00,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8d02dbe582cd4579b611e91d6471c51a, entries=150, sequenceid=333, filesize=12.0 K 2024-11-19T16:17:00,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/a904de6e0f2842bca9d148f2aca012d6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a904de6e0f2842bca9d148f2aca012d6 2024-11-19T16:17:00,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a904de6e0f2842bca9d148f2aca012d6, entries=150, sequenceid=333, filesize=12.0 K 2024-11-19T16:17:00,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0dc20a97b5ebf3ba62589849ed17c46f in 1284ms, sequenceid=333, compaction requested=true 2024-11-19T16:17:00,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:00,633 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:17:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:17:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:17:00,634 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:17:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:17:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:17:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:17:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:17:00,637 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:17:00,637 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:17:00,637 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/B is initiating minor compaction (all files) 2024-11-19T16:17:00,637 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/A is initiating minor compaction (all files) 2024-11-19T16:17:00,637 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/A in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:00,637 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/B in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:00,637 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/87317cbc64c04c16abf7114410d29039, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ffb945437dea40849b142e149275cb96, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22eed6384e5c4e419ee2e549dfc87bac, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/d7b84eb8b287488db151f75ff9ad3abc] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=131.0 K 2024-11-19T16:17:00,637 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5c619bec28cb4cf0aa15098f8f99fbcc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/82189379da834333b255dc45a9e604b6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/23bfe436a74b4329af5920a3851770cb, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8d02dbe582cd4579b611e91d6471c51a] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=48.5 K 2024-11-19T16:17:00,637 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:00,637 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/87317cbc64c04c16abf7114410d29039, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ffb945437dea40849b142e149275cb96, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22eed6384e5c4e419ee2e549dfc87bac, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/d7b84eb8b287488db151f75ff9ad3abc] 2024-11-19T16:17:00,638 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c619bec28cb4cf0aa15098f8f99fbcc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732033015815 2024-11-19T16:17:00,638 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87317cbc64c04c16abf7114410d29039, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732033015815 2024-11-19T16:17:00,639 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 82189379da834333b255dc45a9e604b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732033016172 2024-11-19T16:17:00,639 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffb945437dea40849b142e149275cb96, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732033016172 2024-11-19T16:17:00,639 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22eed6384e5c4e419ee2e549dfc87bac, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732033017439 2024-11-19T16:17:00,639 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 23bfe436a74b4329af5920a3851770cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732033017439 2024-11-19T16:17:00,639 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7b84eb8b287488db151f75ff9ad3abc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732033018185 2024-11-19T16:17:00,639 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d02dbe582cd4579b611e91d6471c51a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732033018201 2024-11-19T16:17:00,652 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:00,657 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411199ae18374f8ae4a0ba8a35fa1cd30cd43_0dc20a97b5ebf3ba62589849ed17c46f store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:00,659 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#B#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:17:00,659 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411199ae18374f8ae4a0ba8a35fa1cd30cd43_0dc20a97b5ebf3ba62589849ed17c46f, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:00,659 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199ae18374f8ae4a0ba8a35fa1cd30cd43_0dc20a97b5ebf3ba62589849ed17c46f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:00,659 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/f964c3bee0f844138c7134f58be68d34 is 50, key is test_row_0/B:col10/1732033019349/Put/seqid=0 2024-11-19T16:17:00,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742442_1618 (size=12915) 2024-11-19T16:17:00,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742443_1619 (size=4469) 2024-11-19T16:17:00,670 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#A#compaction#522 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:17:00,671 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/efbf483ee06146acb04e84a6e6608bb6 is 175, key is test_row_0/A:col10/1732033019349/Put/seqid=0 2024-11-19T16:17:00,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742444_1620 (size=31869) 2024-11-19T16:17:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T16:17:00,767 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:17:00,767 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:17:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-19T16:17:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T16:17:00,769 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:17:00,769 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:17:00,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:17:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T16:17:00,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T16:17:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:00,921 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T16:17:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:17:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:17:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:17:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:00,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f9f25494d81d4171bb37940b1016b057_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033019384/Put/seqid=0 2024-11-19T16:17:00,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742445_1621 (size=12454) 2024-11-19T16:17:00,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:00,933 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f9f25494d81d4171bb37940b1016b057_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f9f25494d81d4171bb37940b1016b057_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:00,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/07ee82faf00843c58d52e788c08fd860, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:00,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/07ee82faf00843c58d52e788c08fd860 is 175, key is test_row_0/A:col10/1732033019384/Put/seqid=0 2024-11-19T16:17:00,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742446_1622 (size=31255) 2024-11-19T16:17:01,072 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/f964c3bee0f844138c7134f58be68d34 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f964c3bee0f844138c7134f58be68d34 2024-11-19T16:17:01,077 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/B of 0dc20a97b5ebf3ba62589849ed17c46f into f964c3bee0f844138c7134f58be68d34(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:17:01,077 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:01,077 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/B, priority=12, startTime=1732033020634; duration=0sec 2024-11-19T16:17:01,077 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:17:01,077 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:B 2024-11-19T16:17:01,077 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:17:01,078 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/efbf483ee06146acb04e84a6e6608bb6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/efbf483ee06146acb04e84a6e6608bb6 2024-11-19T16:17:01,080 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:17:01,080 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/C is initiating minor compaction (all files) 2024-11-19T16:17:01,080 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/C in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:01,080 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/8d671518c60d47f4911bb28afebbf5c3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a408e7591c4e459d8eefcd47b920ad34, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/ef1ef1cc0c1647e3bb041ce4ce2f80db, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a904de6e0f2842bca9d148f2aca012d6] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=48.5 K 2024-11-19T16:17:01,080 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d671518c60d47f4911bb28afebbf5c3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732033015815 2024-11-19T16:17:01,081 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a408e7591c4e459d8eefcd47b920ad34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732033016172 2024-11-19T16:17:01,081 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting ef1ef1cc0c1647e3bb041ce4ce2f80db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732033017439 2024-11-19T16:17:01,081 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a904de6e0f2842bca9d148f2aca012d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732033018201 2024-11-19T16:17:01,083 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/A of 0dc20a97b5ebf3ba62589849ed17c46f into efbf483ee06146acb04e84a6e6608bb6(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:17:01,083 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:01,083 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/A, priority=12, startTime=1732033020633; duration=0sec 2024-11-19T16:17:01,083 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:17:01,083 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:A 2024-11-19T16:17:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T16:17:01,092 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#C#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:17:01,092 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/ddcb13a4f8e2456cb6b009ecc63da38f is 50, key is test_row_0/C:col10/1732033019349/Put/seqid=0 2024-11-19T16:17:01,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742447_1623 (size=12915) 2024-11-19T16:17:01,338 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/07ee82faf00843c58d52e788c08fd860 2024-11-19T16:17:01,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/a3b42e0a3a2242068bf99724fb117e3a is 50, key is test_row_0/B:col10/1732033019384/Put/seqid=0 2024-11-19T16:17:01,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742448_1624 (size=12301) 2024-11-19T16:17:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T16:17:01,510 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/ddcb13a4f8e2456cb6b009ecc63da38f as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/ddcb13a4f8e2456cb6b009ecc63da38f 2024-11-19T16:17:01,514 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/C of 0dc20a97b5ebf3ba62589849ed17c46f into ddcb13a4f8e2456cb6b009ecc63da38f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:17:01,514 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:01,514 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/C, priority=12, startTime=1732033020634; duration=0sec 2024-11-19T16:17:01,514 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:17:01,514 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:C 2024-11-19T16:17:01,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:01,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033081625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:01,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033081628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:01,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033081736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:01,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033081737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:01,754 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/a3b42e0a3a2242068bf99724fb117e3a 2024-11-19T16:17:01,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/92d1d3d4251c4b01855232dcbbc06b53 is 50, key is test_row_0/C:col10/1732033019384/Put/seqid=0 2024-11-19T16:17:01,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742449_1625 (size=12301) 2024-11-19T16:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T16:17:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033081946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:01,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033081946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:02,170 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/92d1d3d4251c4b01855232dcbbc06b53 2024-11-19T16:17:02,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/07ee82faf00843c58d52e788c08fd860 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/07ee82faf00843c58d52e788c08fd860 2024-11-19T16:17:02,179 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/07ee82faf00843c58d52e788c08fd860, entries=150, sequenceid=345, filesize=30.5 K 2024-11-19T16:17:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/a3b42e0a3a2242068bf99724fb117e3a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/a3b42e0a3a2242068bf99724fb117e3a 2024-11-19T16:17:02,183 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/a3b42e0a3a2242068bf99724fb117e3a, entries=150, sequenceid=345, filesize=12.0 K 2024-11-19T16:17:02,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/92d1d3d4251c4b01855232dcbbc06b53 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/92d1d3d4251c4b01855232dcbbc06b53 2024-11-19T16:17:02,186 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/92d1d3d4251c4b01855232dcbbc06b53, entries=150, sequenceid=345, filesize=12.0 K 2024-11-19T16:17:02,187 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0dc20a97b5ebf3ba62589849ed17c46f in 1266ms, sequenceid=345, compaction requested=false 2024-11-19T16:17:02,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:02,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:02,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-19T16:17:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-19T16:17:02,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-19T16:17:02,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4190 sec 2024-11-19T16:17:02,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4260 sec 2024-11-19T16:17:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:02,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T16:17:02,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:17:02,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:17:02,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:17:02,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:17:02,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:17:02,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T16:17:02,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b0a52c3b05244a7b96d37ad0a5cebd7d_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033022258/Put/seqid=0 2024-11-19T16:17:02,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742450_1626 (size=12454) 2024-11-19T16:17:02,276 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,282 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b0a52c3b05244a7b96d37ad0a5cebd7d_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b0a52c3b05244a7b96d37ad0a5cebd7d_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:02,283 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/da7cafb4f16445568ca2c8d2dcb04205, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:02,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/da7cafb4f16445568ca2c8d2dcb04205 is 175, key is test_row_0/A:col10/1732033022258/Put/seqid=0 2024-11-19T16:17:02,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742451_1627 (size=31255) 2024-11-19T16:17:02,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033082285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:02,291 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/da7cafb4f16445568ca2c8d2dcb04205 2024-11-19T16:17:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:02,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033082286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:02,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/d1992a7f13d340499433905d5435b9f6 is 50, key is test_row_0/B:col10/1732033022258/Put/seqid=0 2024-11-19T16:17:02,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742452_1628 (size=12301) 2024-11-19T16:17:02,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/d1992a7f13d340499433905d5435b9f6 2024-11-19T16:17:02,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/e4f0159465344c739f9403a18f05f560 is 50, key is test_row_0/C:col10/1732033022258/Put/seqid=0 2024-11-19T16:17:02,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742453_1629 (size=12301) 2024-11-19T16:17:02,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/e4f0159465344c739f9403a18f05f560 2024-11-19T16:17:02,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/da7cafb4f16445568ca2c8d2dcb04205 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/da7cafb4f16445568ca2c8d2dcb04205 2024-11-19T16:17:02,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/da7cafb4f16445568ca2c8d2dcb04205, entries=150, sequenceid=369, filesize=30.5 K 2024-11-19T16:17:02,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/d1992a7f13d340499433905d5435b9f6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/d1992a7f13d340499433905d5435b9f6 2024-11-19T16:17:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/d1992a7f13d340499433905d5435b9f6, entries=150, sequenceid=369, filesize=12.0 K 2024-11-19T16:17:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/e4f0159465344c739f9403a18f05f560 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/e4f0159465344c739f9403a18f05f560 2024-11-19T16:17:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/e4f0159465344c739f9403a18f05f560, entries=150, sequenceid=369, filesize=12.0 K 2024-11-19T16:17:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 0dc20a97b5ebf3ba62589849ed17c46f in 89ms, sequenceid=369, compaction requested=true 2024-11-19T16:17:02,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:02,349 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:17:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:17:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:17:02,349 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:17:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:17:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:17:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:17:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:17:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,349 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:17:02,349 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/A is initiating minor compaction (all files) 2024-11-19T16:17:02,349 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/A in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,349 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/efbf483ee06146acb04e84a6e6608bb6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/07ee82faf00843c58d52e788c08fd860, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/da7cafb4f16445568ca2c8d2dcb04205] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=92.2 K 2024-11-19T16:17:02,350 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:02,350 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/efbf483ee06146acb04e84a6e6608bb6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/07ee82faf00843c58d52e788c08fd860, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/da7cafb4f16445568ca2c8d2dcb04205] 2024-11-19T16:17:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,350 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:17:02,350 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/B is initiating minor compaction (all files) 2024-11-19T16:17:02,350 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/B in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:02,350 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f964c3bee0f844138c7134f58be68d34, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/a3b42e0a3a2242068bf99724fb117e3a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/d1992a7f13d340499433905d5435b9f6] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=36.6 K 2024-11-19T16:17:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,350 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting efbf483ee06146acb04e84a6e6608bb6, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732033018201 2024-11-19T16:17:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,351 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07ee82faf00843c58d52e788c08fd860, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732033019384 2024-11-19T16:17:02,351 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting da7cafb4f16445568ca2c8d2dcb04205, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732033021622 2024-11-19T16:17:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,351 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting f964c3bee0f844138c7134f58be68d34, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732033018201 2024-11-19T16:17:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,352 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting a3b42e0a3a2242068bf99724fb117e3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732033019384 2024-11-19T16:17:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,352 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting d1992a7f13d340499433905d5435b9f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732033021622 2024-11-19T16:17:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,357 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,361 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119f13b3e8e37ab493fb5c4f213413b5aa3_0dc20a97b5ebf3ba62589849ed17c46f store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:02,362 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#B#compaction#532 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:17:02,363 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/59e65446ccff462d818e8b96a891062e is 50, key is test_row_0/B:col10/1732033022258/Put/seqid=0 2024-11-19T16:17:02,363 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119f13b3e8e37ab493fb5c4f213413b5aa3_0dc20a97b5ebf3ba62589849ed17c46f, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:02,363 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f13b3e8e37ab493fb5c4f213413b5aa3_0dc20a97b5ebf3ba62589849ed17c46f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742454_1630 (size=13017) 2024-11-19T16:17:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742455_1631 (size=4469) 2024-11-19T16:17:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,385 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/59e65446ccff462d818e8b96a891062e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/59e65446ccff462d818e8b96a891062e 2024-11-19T16:17:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,388 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/B of 0dc20a97b5ebf3ba62589849ed17c46f into 59e65446ccff462d818e8b96a891062e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:17:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,388 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:02,388 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/B, priority=13, startTime=1732033022349; duration=0sec 2024-11-19T16:17:02,388 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:17:02,388 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:B 2024-11-19T16:17:02,388 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T16:17:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,389 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T16:17:02,389 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/C is initiating minor compaction (all files) 2024-11-19T16:17:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,389 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/C in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:02,389 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/ddcb13a4f8e2456cb6b009ecc63da38f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/92d1d3d4251c4b01855232dcbbc06b53, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/e4f0159465344c739f9403a18f05f560] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=36.6 K 2024-11-19T16:17:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,389 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting ddcb13a4f8e2456cb6b009ecc63da38f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732033018201 2024-11-19T16:17:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,390 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 92d1d3d4251c4b01855232dcbbc06b53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732033019384 2024-11-19T16:17:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,390 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting e4f0159465344c739f9403a18f05f560, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732033021622 2024-11-19T16:17:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,396 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#C#compaction#533 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:17:02,396 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/039ef975ab194d16845decdb6d9f5150 is 50, key is test_row_0/C:col10/1732033022258/Put/seqid=0 2024-11-19T16:17:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742456_1632 (size=13017) 2024-11-19T16:17:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T16:17:02,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:17:02,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:02,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:17:02,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:17:02,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a2025a01d42540f59e18aed9db166588_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033022400/Put/seqid=0 2024-11-19T16:17:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742457_1633 (size=14994) 2024-11-19T16:17:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:02,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033082458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:02,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:02,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033082460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033082567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033082567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:02,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033082777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033082777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:02,783 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#A#compaction#531 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:17:02,784 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/501d73e3b14e4cbb997e34912453243e is 175, key is test_row_0/A:col10/1732033022258/Put/seqid=0 2024-11-19T16:17:02,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742458_1634 (size=31971) 2024-11-19T16:17:02,802 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/039ef975ab194d16845decdb6d9f5150 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/039ef975ab194d16845decdb6d9f5150 2024-11-19T16:17:02,807 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/C of 0dc20a97b5ebf3ba62589849ed17c46f into 039ef975ab194d16845decdb6d9f5150(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:17:02,807 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:02,807 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/C, priority=13, startTime=1732033022349; duration=0sec 2024-11-19T16:17:02,807 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:17:02,807 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:C 2024-11-19T16:17:02,818 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:02,824 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a2025a01d42540f59e18aed9db166588_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a2025a01d42540f59e18aed9db166588_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:02,825 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/f2ed7c46c0fb448e82b8f6f45f835e54, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:02,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/f2ed7c46c0fb448e82b8f6f45f835e54 is 175, key is test_row_0/A:col10/1732033022400/Put/seqid=0 2024-11-19T16:17:02,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742459_1635 (size=39945) 2024-11-19T16:17:02,828 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/f2ed7c46c0fb448e82b8f6f45f835e54 2024-11-19T16:17:02,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/25360e70f1a2421b80042053722372b3 is 50, key is test_row_0/B:col10/1732033022400/Put/seqid=0 2024-11-19T16:17:02,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742460_1636 (size=9857) 2024-11-19T16:17:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T16:17:02,917 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:17:02,917 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:17:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-19T16:17:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T16:17:02,919 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:17:02,919 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:17:02,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:17:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T16:17:03,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T16:17:03,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:03,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:03,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:03,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:03,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:03,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033083087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:03,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033083087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:03,191 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/501d73e3b14e4cbb997e34912453243e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/501d73e3b14e4cbb997e34912453243e 2024-11-19T16:17:03,194 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/A of 0dc20a97b5ebf3ba62589849ed17c46f into 501d73e3b14e4cbb997e34912453243e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:17:03,194 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:03,194 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/A, priority=13, startTime=1732033022348; duration=0sec 2024-11-19T16:17:03,195 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:17:03,195 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:A 2024-11-19T16:17:03,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T16:17:03,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:03,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:03,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:03,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:03,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T16:17:03,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/25360e70f1a2421b80042053722372b3 2024-11-19T16:17:03,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/b72e9459642b481bbd2e4f385cec433e is 50, key is test_row_0/C:col10/1732033022400/Put/seqid=0 2024-11-19T16:17:03,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742461_1637 (size=9857) 2024-11-19T16:17:03,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T16:17:03,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:03,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:03,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:03,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:03,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:03,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T16:17:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:03,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T16:17:03,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:03,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033083597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033083597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:03,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/b72e9459642b481bbd2e4f385cec433e 2024-11-19T16:17:03,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/f2ed7c46c0fb448e82b8f6f45f835e54 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/f2ed7c46c0fb448e82b8f6f45f835e54 2024-11-19T16:17:03,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/f2ed7c46c0fb448e82b8f6f45f835e54, entries=200, sequenceid=383, filesize=39.0 K 2024-11-19T16:17:03,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/25360e70f1a2421b80042053722372b3 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/25360e70f1a2421b80042053722372b3 2024-11-19T16:17:03,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/25360e70f1a2421b80042053722372b3, entries=100, sequenceid=383, filesize=9.6 K 2024-11-19T16:17:03,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/b72e9459642b481bbd2e4f385cec433e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/b72e9459642b481bbd2e4f385cec433e 2024-11-19T16:17:03,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/b72e9459642b481bbd2e4f385cec433e, entries=100, sequenceid=383, filesize=9.6 K 2024-11-19T16:17:03,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0dc20a97b5ebf3ba62589849ed17c46f in 1271ms, sequenceid=383, compaction requested=false 2024-11-19T16:17:03,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:03,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T16:17:03,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:03,680 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T16:17:03,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:17:03,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:03,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:17:03,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:03,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:17:03,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:03,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111911a23a8098ac4453926c18131aa5e214_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033022457/Put/seqid=0 2024-11-19T16:17:03,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742462_1638 (size=12454) 2024-11-19T16:17:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T16:17:04,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:04,102 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111911a23a8098ac4453926c18131aa5e214_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111911a23a8098ac4453926c18131aa5e214_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0c43bc3c89ff4f8596a97256c5ae70e0, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0c43bc3c89ff4f8596a97256c5ae70e0 is 175, key is test_row_0/A:col10/1732033022457/Put/seqid=0 2024-11-19T16:17:04,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742463_1639 (size=31255) 2024-11-19T16:17:04,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:04,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:04,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033084380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:04,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:04,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033084385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:04,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:04,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033084386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:04,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033084497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:04,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033084497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:04,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033084497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:04,520 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0c43bc3c89ff4f8596a97256c5ae70e0 2024-11-19T16:17:04,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/5dc541f5ccc148be804082f9e3cf5aa4 is 50, key is test_row_0/B:col10/1732033022457/Put/seqid=0 2024-11-19T16:17:04,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742464_1640 (size=12301) 2024-11-19T16:17:04,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033084607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:04,610 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 0dc20a97b5ebf3ba62589849ed17c46f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:17:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:04,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033084617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:04,619 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_0' in 0dc20a97b5ebf3ba62589849ed17c46f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:17:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:04,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033084707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:04,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:04,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033084707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033084707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:04,930 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/5dc541f5ccc148be804082f9e3cf5aa4 2024-11-19T16:17:04,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/31436a8f263d42048f848320499f62f1 is 50, key is test_row_0/C:col10/1732033022457/Put/seqid=0 2024-11-19T16:17:04,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742465_1641 (size=12301) 2024-11-19T16:17:04,940 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/31436a8f263d42048f848320499f62f1 2024-11-19T16:17:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0c43bc3c89ff4f8596a97256c5ae70e0 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0c43bc3c89ff4f8596a97256c5ae70e0 2024-11-19T16:17:04,950 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0c43bc3c89ff4f8596a97256c5ae70e0, entries=150, sequenceid=404, filesize=30.5 K 2024-11-19T16:17:04,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/5dc541f5ccc148be804082f9e3cf5aa4 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5dc541f5ccc148be804082f9e3cf5aa4 2024-11-19T16:17:04,953 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5dc541f5ccc148be804082f9e3cf5aa4, entries=150, sequenceid=404, filesize=12.0 K 2024-11-19T16:17:04,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/31436a8f263d42048f848320499f62f1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/31436a8f263d42048f848320499f62f1 2024-11-19T16:17:04,959 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/31436a8f263d42048f848320499f62f1, entries=150, sequenceid=404, filesize=12.0 K 2024-11-19T16:17:04,960 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0dc20a97b5ebf3ba62589849ed17c46f in 1279ms, sequenceid=404, compaction requested=true 2024-11-19T16:17:04,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:04,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:04,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-19T16:17:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-19T16:17:04,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-19T16:17:04,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0420 sec 2024-11-19T16:17:04,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.0470 sec 2024-11-19T16:17:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:05,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T16:17:05,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:17:05,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:05,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:17:05,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:05,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:17:05,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:05,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111910441db17e394799976a644b04d8c77c_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033025016/Put/seqid=0 2024-11-19T16:17:05,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742466_1642 (size=14994) 2024-11-19T16:17:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033085042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:05,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033085043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:05,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033085046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T16:17:05,067 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:17:05,068 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T16:17:05,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-19T16:17:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T16:17:05,069 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T16:17:05,070 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T16:17:05,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T16:17:05,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033085157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:05,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033085157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033085157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T16:17:05,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T16:17:05,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:05,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:05,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:05,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033085367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:05,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033085367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:05,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033085367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:05,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T16:17:05,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:05,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:05,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:05,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T16:17:05,427 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:05,429 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111910441db17e394799976a644b04d8c77c_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111910441db17e394799976a644b04d8c77c_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:05,430 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0d1e0b472d454d708a38cf951912f7e6, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:05,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0d1e0b472d454d708a38cf951912f7e6 is 175, key is test_row_0/A:col10/1732033025016/Put/seqid=0 2024-11-19T16:17:05,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742467_1643 (size=39949) 2024-11-19T16:17:05,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T16:17:05,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:05,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:05,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:05,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T16:17:05,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:05,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:05,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:05,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033085677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033085677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:05,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033085677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T16:17:05,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T16:17:05,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:05,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,838 INFO [Thread-2446 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2446. 2024-11-19T16:17:05,838 DEBUG [Thread-2446 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:17:05,838 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:05,838 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:05,838 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=418, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0d1e0b472d454d708a38cf951912f7e6 2024-11-19T16:17:05,839 INFO [Thread-2438 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2438. 2024-11-19T16:17:05,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:17:05,839 DEBUG [Thread-2438 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:17:05,839 DEBUG [Thread-2438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:05,839 DEBUG [Thread-2438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:05,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:17:05,841 INFO [Thread-2442 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2442. 2024-11-19T16:17:05,841 DEBUG [Thread-2442 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:17:05,841 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:05,841 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:05,841 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:17:05,842 INFO [Thread-2444 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2444. 2024-11-19T16:17:05,842 DEBUG [Thread-2444 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:17:05,842 DEBUG [Thread-2444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:05,842 INFO [Thread-2440 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2440. 2024-11-19T16:17:05,842 DEBUG [Thread-2440 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:17:05,842 DEBUG [Thread-2444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:05,842 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:05,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:17:05,843 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:05,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:17:05,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/8771e30c7d814114bd8b56bb072fa41e is 50, key is test_row_0/B:col10/1732033025016/Put/seqid=0 2024-11-19T16:17:05,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742468_1644 (size=12301) 2024-11-19T16:17:05,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T16:17:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:05,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:06,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T16:17:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:06,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:06,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:06,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33094 deadline: 1732033086187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732033086187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:06,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33140 deadline: 1732033086197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T16:17:06,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/8771e30c7d814114bd8b56bb072fa41e 2024-11-19T16:17:06,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/cc0db331b2eb4e5e9dc3169115fc3470 is 50, key is test_row_0/C:col10/1732033025016/Put/seqid=0 2024-11-19T16:17:06,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742469_1645 (size=12301) 2024-11-19T16:17:06,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T16:17:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:06,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:06,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T16:17:06,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:06,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:06,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:06,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:06,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:06,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T16:17:06,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:06,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:06,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:06,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:06,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T16:17:06,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33118 deadline: 1732033086626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:06,627 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_2' in 0dc20a97b5ebf3ba62589849ed17c46f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:17:06,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:06,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:06,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:06,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T16:17:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33102 deadline: 1732033086637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:06,638 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32a8860bb9d6,43095,1732032854050 for 'test_row_0' in 0dc20a97b5ebf3ba62589849ed17c46f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T16:17:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 , the old value is region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T16:17:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., hostname=32a8860bb9d6,43095,1732032854050, seqNum=5 because the exception is null or not the one we care about 2024-11-19T16:17:06,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/cc0db331b2eb4e5e9dc3169115fc3470 2024-11-19T16:17:06,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/0d1e0b472d454d708a38cf951912f7e6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d1e0b472d454d708a38cf951912f7e6 2024-11-19T16:17:06,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d1e0b472d454d708a38cf951912f7e6, entries=200, sequenceid=418, filesize=39.0 K 2024-11-19T16:17:06,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/8771e30c7d814114bd8b56bb072fa41e as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8771e30c7d814114bd8b56bb072fa41e 2024-11-19T16:17:06,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8771e30c7d814114bd8b56bb072fa41e, entries=150, sequenceid=418, filesize=12.0 K 2024-11-19T16:17:06,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/cc0db331b2eb4e5e9dc3169115fc3470 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/cc0db331b2eb4e5e9dc3169115fc3470 2024-11-19T16:17:06,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/cc0db331b2eb4e5e9dc3169115fc3470, entries=150, sequenceid=418, filesize=12.0 K 2024-11-19T16:17:06,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0dc20a97b5ebf3ba62589849ed17c46f in 1661ms, sequenceid=418, compaction requested=true 2024-11-19T16:17:06,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:06,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T16:17:06,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:17:06,679 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:17:06,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T16:17:06,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:17:06,679 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:17:06,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0dc20a97b5ebf3ba62589849ed17c46f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T16:17:06,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:17:06,679 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:17:06,679 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143120 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:17:06,679 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/B is initiating minor compaction (all files) 2024-11-19T16:17:06,679 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/A is initiating minor compaction (all files) 2024-11-19T16:17:06,680 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/B in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:06,680 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/A in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:06,680 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/59e65446ccff462d818e8b96a891062e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/25360e70f1a2421b80042053722372b3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5dc541f5ccc148be804082f9e3cf5aa4, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8771e30c7d814114bd8b56bb072fa41e] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=46.4 K 2024-11-19T16:17:06,680 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/501d73e3b14e4cbb997e34912453243e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/f2ed7c46c0fb448e82b8f6f45f835e54, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0c43bc3c89ff4f8596a97256c5ae70e0, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d1e0b472d454d708a38cf951912f7e6] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=139.8 K 2024-11-19T16:17:06,680 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:06,680 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. files: [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/501d73e3b14e4cbb997e34912453243e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/f2ed7c46c0fb448e82b8f6f45f835e54, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0c43bc3c89ff4f8596a97256c5ae70e0, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d1e0b472d454d708a38cf951912f7e6] 2024-11-19T16:17:06,680 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 501d73e3b14e4cbb997e34912453243e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732033021622 2024-11-19T16:17:06,680 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 59e65446ccff462d818e8b96a891062e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732033021622 2024-11-19T16:17:06,681 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 25360e70f1a2421b80042053722372b3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732033022399 2024-11-19T16:17:06,681 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2ed7c46c0fb448e82b8f6f45f835e54, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732033022282 2024-11-19T16:17:06,681 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dc541f5ccc148be804082f9e3cf5aa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732033022452 2024-11-19T16:17:06,681 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c43bc3c89ff4f8596a97256c5ae70e0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732033022452 2024-11-19T16:17:06,681 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 8771e30c7d814114bd8b56bb072fa41e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732033024365 2024-11-19T16:17:06,681 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d1e0b472d454d708a38cf951912f7e6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732033024365 2024-11-19T16:17:06,685 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:06,687 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#B#compaction#543 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:17:06,687 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119737b00caee134aab89df0bba4bcc97c1_0dc20a97b5ebf3ba62589849ed17c46f store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:06,687 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/491f95853aeb4b45b12cc1a605daf1a7 is 50, key is test_row_0/B:col10/1732033025016/Put/seqid=0 2024-11-19T16:17:06,690 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119737b00caee134aab89df0bba4bcc97c1_0dc20a97b5ebf3ba62589849ed17c46f, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:06,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742470_1646 (size=13153) 2024-11-19T16:17:06,691 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119737b00caee134aab89df0bba4bcc97c1_0dc20a97b5ebf3ba62589849ed17c46f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:06,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742471_1647 (size=4469) 2024-11-19T16:17:06,694 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#A#compaction#544 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:17:06,695 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/491f95853aeb4b45b12cc1a605daf1a7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/491f95853aeb4b45b12cc1a605daf1a7 2024-11-19T16:17:06,695 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/a5a290649eb54f958f4b8644a97421ee is 175, key is test_row_0/A:col10/1732033025016/Put/seqid=0 2024-11-19T16:17:06,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742472_1648 (size=32107) 2024-11-19T16:17:06,699 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/B of 0dc20a97b5ebf3ba62589849ed17c46f into 491f95853aeb4b45b12cc1a605daf1a7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:17:06,699 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:06,699 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/B, priority=12, startTime=1732033026679; duration=0sec 2024-11-19T16:17:06,699 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T16:17:06,699 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:B 2024-11-19T16:17:06,699 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T16:17:06,700 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T16:17:06,700 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1541): 0dc20a97b5ebf3ba62589849ed17c46f/C is initiating minor compaction (all files) 2024-11-19T16:17:06,700 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0dc20a97b5ebf3ba62589849ed17c46f/C in TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:06,700 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/039ef975ab194d16845decdb6d9f5150, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/b72e9459642b481bbd2e4f385cec433e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/31436a8f263d42048f848320499f62f1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/cc0db331b2eb4e5e9dc3169115fc3470] into tmpdir=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp, totalSize=46.4 K 2024-11-19T16:17:06,700 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 039ef975ab194d16845decdb6d9f5150, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732033021622 2024-11-19T16:17:06,700 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting b72e9459642b481bbd2e4f385cec433e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732033022399 2024-11-19T16:17:06,701 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting 31436a8f263d42048f848320499f62f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732033022452 2024-11-19T16:17:06,701 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] compactions.Compactor(225): Compacting cc0db331b2eb4e5e9dc3169115fc3470, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732033024365 2024-11-19T16:17:06,701 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/a5a290649eb54f958f4b8644a97421ee as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/a5a290649eb54f958f4b8644a97421ee 2024-11-19T16:17:06,705 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/A of 0dc20a97b5ebf3ba62589849ed17c46f into a5a290649eb54f958f4b8644a97421ee(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:17:06,705 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:06,705 INFO [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/A, priority=12, startTime=1732033026679; duration=0sec 2024-11-19T16:17:06,705 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:17:06,705 DEBUG [RS:0;32a8860bb9d6:43095-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:A 2024-11-19T16:17:06,708 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0dc20a97b5ebf3ba62589849ed17c46f#C#compaction#545 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T16:17:06,708 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/55c172c2b4ca4c34b0053a67cb1d6193 is 50, key is test_row_0/C:col10/1732033025016/Put/seqid=0 2024-11-19T16:17:06,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742473_1649 (size=13153) 2024-11-19T16:17:06,714 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/55c172c2b4ca4c34b0053a67cb1d6193 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/55c172c2b4ca4c34b0053a67cb1d6193 2024-11-19T16:17:06,718 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0dc20a97b5ebf3ba62589849ed17c46f/C of 0dc20a97b5ebf3ba62589849ed17c46f into 55c172c2b4ca4c34b0053a67cb1d6193(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T16:17:06,718 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:06,718 INFO [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., storeName=0dc20a97b5ebf3ba62589849ed17c46f/C, priority=12, startTime=1732033026679; duration=0sec 2024-11-19T16:17:06,718 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T16:17:06,718 DEBUG [RS:0;32a8860bb9d6:43095-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0dc20a97b5ebf3ba62589849ed17c46f:C 2024-11-19T16:17:06,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T16:17:06,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:06,756 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T16:17:06,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:17:06,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:06,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:17:06,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:06,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:17:06,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:06,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195d624e6464cf4ed594893955b5370d4a_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033025042/Put/seqid=0 2024-11-19T16:17:06,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742474_1650 (size=12454) 2024-11-19T16:17:07,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:07,170 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195d624e6464cf4ed594893955b5370d4a_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195d624e6464cf4ed594893955b5370d4a_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:07,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/da6b022a90b94ec280598089998613a6, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:07,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/da6b022a90b94ec280598089998613a6 is 175, key is test_row_0/A:col10/1732033025042/Put/seqid=0 2024-11-19T16:17:07,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742475_1651 (size=31255) 2024-11-19T16:17:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43095 {}] regionserver.HRegion(8855): Flush requested on 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:07,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. as already flushing 2024-11-19T16:17:07,200 INFO [Thread-2431 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2431. 2024-11-19T16:17:07,200 DEBUG [Thread-2431 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:17:07,200 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:07,200 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:07,200 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:17:07,209 INFO [Thread-2435 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2435. 2024-11-19T16:17:07,209 INFO [Thread-2429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2429. 2024-11-19T16:17:07,209 DEBUG [Thread-2435 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:17:07,209 DEBUG [Thread-2429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:17:07,209 DEBUG [Thread-2435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:07,209 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:07,209 DEBUG [Thread-2435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:07,209 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:07,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:17:07,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:17:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T16:17:07,575 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=439, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/da6b022a90b94ec280598089998613a6 2024-11-19T16:17:07,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/5fc4ca6271414b8cbb0326fe77566416 is 50, key is test_row_0/B:col10/1732033025042/Put/seqid=0 2024-11-19T16:17:07,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742476_1652 (size=12301) 2024-11-19T16:17:07,988 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/5fc4ca6271414b8cbb0326fe77566416 2024-11-19T16:17:07,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/da97942a2ed1484eaa8c6ea7292e0c0c is 50, key is test_row_0/C:col10/1732033025042/Put/seqid=0 2024-11-19T16:17:08,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742477_1653 (size=12301) 2024-11-19T16:17:08,402 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/da97942a2ed1484eaa8c6ea7292e0c0c 2024-11-19T16:17:08,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/da6b022a90b94ec280598089998613a6 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/da6b022a90b94ec280598089998613a6 2024-11-19T16:17:08,412 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/da6b022a90b94ec280598089998613a6, entries=150, sequenceid=439, filesize=30.5 K 2024-11-19T16:17:08,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/5fc4ca6271414b8cbb0326fe77566416 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5fc4ca6271414b8cbb0326fe77566416 2024-11-19T16:17:08,416 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5fc4ca6271414b8cbb0326fe77566416, entries=150, sequenceid=439, filesize=12.0 K 2024-11-19T16:17:08,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/da97942a2ed1484eaa8c6ea7292e0c0c as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/da97942a2ed1484eaa8c6ea7292e0c0c 2024-11-19T16:17:08,421 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/da97942a2ed1484eaa8c6ea7292e0c0c, entries=150, sequenceid=439, filesize=12.0 K 2024-11-19T16:17:08,421 INFO [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=20.13 KB/20610 for 0dc20a97b5ebf3ba62589849ed17c46f in 1665ms, sequenceid=439, compaction requested=false 2024-11-19T16:17:08,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 0dc20a97b5ebf3ba62589849ed17c46f: 2024-11-19T16:17:08,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:08,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32a8860bb9d6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-19T16:17:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-19T16:17:08,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-19T16:17:08,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3530 sec 2024-11-19T16:17:08,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 3.3570 sec 2024-11-19T16:17:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T16:17:09,248 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:17:10,638 INFO [Thread-2427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2427. 2024-11-19T16:17:10,638 DEBUG [Thread-2427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:17:10,638 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:10,638 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:10,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:17:10,649 INFO [Thread-2433 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2433. 2024-11-19T16:17:10,649 DEBUG [Thread-2433 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T16:17:10,649 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:10,649 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:10,649 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:17:10,650 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T16:17:10,650 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-19T16:17:10,650 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-19T16:17:10,650 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-19T16:17:10,650 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-19T16:17:10,650 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-19T16:17:10,650 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T16:17:10,650 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2811 2024-11-19T16:17:10,650 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2871 2024-11-19T16:17:10,650 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2892 2024-11-19T16:17:10,650 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2827 2024-11-19T16:17:10,650 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2864 2024-11-19T16:17:10,650 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T16:17:10,650 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T16:17:10,650 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T16:17:10,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:10,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:10,651 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:17:10,652 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-19T16:17:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T16:17:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T16:17:10,657 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732033030656"}]},"ts":"1732033030656"} 2024-11-19T16:17:10,658 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T16:17:10,658 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T16:17:10,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T16:17:10,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, UNASSIGN}] 2024-11-19T16:17:10,660 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, UNASSIGN 2024-11-19T16:17:10,661 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=0dc20a97b5ebf3ba62589849ed17c46f, regionState=CLOSING, regionLocation=32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:10,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, UNASSIGN because future has completed 2024-11-19T16:17:10,662 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T16:17:10,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050}] 2024-11-19T16:17:10,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T16:17:10,815 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:10,815 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T16:17:10,815 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 0dc20a97b5ebf3ba62589849ed17c46f, disabling compactions & flushes 2024-11-19T16:17:10,815 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:10,815 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:10,815 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. after waiting 0 ms 2024-11-19T16:17:10,815 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:10,815 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 0dc20a97b5ebf3ba62589849ed17c46f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T16:17:10,816 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=A 2024-11-19T16:17:10,816 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:10,816 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=B 2024-11-19T16:17:10,816 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:10,816 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0dc20a97b5ebf3ba62589849ed17c46f, store=C 2024-11-19T16:17:10,816 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T16:17:10,824 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198b1bf15b2da14d9588cbb4602a3c5dae_0dc20a97b5ebf3ba62589849ed17c46f is 50, key is test_row_0/A:col10/1732033030647/Put/seqid=0 2024-11-19T16:17:10,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742478_1654 (size=9914) 2024-11-19T16:17:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T16:17:11,229 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T16:17:11,267 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198b1bf15b2da14d9588cbb4602a3c5dae_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198b1bf15b2da14d9588cbb4602a3c5dae_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:11,269 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/32f29aa88d184c5c854e1e7f8cb2c290, store: [table=TestAcidGuarantees family=A region=0dc20a97b5ebf3ba62589849ed17c46f] 2024-11-19T16:17:11,270 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/32f29aa88d184c5c854e1e7f8cb2c290 is 175, key is test_row_0/A:col10/1732033030647/Put/seqid=0 2024-11-19T16:17:11,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742479_1655 (size=22561) 2024-11-19T16:17:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T16:17:11,675 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=447, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/32f29aa88d184c5c854e1e7f8cb2c290 2024-11-19T16:17:11,682 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/7dddb94d78e44beea2c19bb059334119 is 50, key is test_row_0/B:col10/1732033030647/Put/seqid=0 2024-11-19T16:17:11,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742480_1656 (size=9857) 2024-11-19T16:17:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T16:17:12,087 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/7dddb94d78e44beea2c19bb059334119 2024-11-19T16:17:12,100 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/faa40a12dec344fa940a717fa9f31f69 is 50, key is test_row_0/C:col10/1732033030647/Put/seqid=0 2024-11-19T16:17:12,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742481_1657 (size=9857) 2024-11-19T16:17:12,301 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T16:17:12,504 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/faa40a12dec344fa940a717fa9f31f69 2024-11-19T16:17:12,509 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/A/32f29aa88d184c5c854e1e7f8cb2c290 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/32f29aa88d184c5c854e1e7f8cb2c290 2024-11-19T16:17:12,513 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/32f29aa88d184c5c854e1e7f8cb2c290, entries=100, sequenceid=447, filesize=22.0 K 2024-11-19T16:17:12,514 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/B/7dddb94d78e44beea2c19bb059334119 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/7dddb94d78e44beea2c19bb059334119 2024-11-19T16:17:12,518 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/7dddb94d78e44beea2c19bb059334119, entries=100, sequenceid=447, filesize=9.6 K 2024-11-19T16:17:12,519 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/.tmp/C/faa40a12dec344fa940a717fa9f31f69 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/faa40a12dec344fa940a717fa9f31f69 2024-11-19T16:17:12,521 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/faa40a12dec344fa940a717fa9f31f69, entries=100, sequenceid=447, filesize=9.6 K 2024-11-19T16:17:12,522 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0dc20a97b5ebf3ba62589849ed17c46f in 1707ms, sequenceid=447, compaction requested=true 2024-11-19T16:17:12,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/e7403480156f4915968799af262b7572, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d0c4d61a0e64e35a4b34e841d7e0069, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/280ef379b0244d6787e204cfb030f41b, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22217a72ff46446f91b5aac4564e0aa7, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/2917e8de0d204f2481147cd77cda80e8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/efc55cda1be0490bb301d3393373b58e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/a0ff963b636d43a8973e77df4edd59d8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ba252afed0394c7e815c8ce8a88d376d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/68ebaa9a7b2d45db91844d697c8b6f8a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0a4b0d72187d4ed3b030cbb1540a7274, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/11567aeacaf242c7960cf8182023fc42, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ea917fef73e847f2b26fd1a27ef0e9c1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/96098c87254940fb8c53f09ad54d9cfe, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/87317cbc64c04c16abf7114410d29039, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/715a2f4afbbb4021a67f8a1dec8a93af, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ffb945437dea40849b142e149275cb96, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22eed6384e5c4e419ee2e549dfc87bac, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/d7b84eb8b287488db151f75ff9ad3abc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/efbf483ee06146acb04e84a6e6608bb6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/07ee82faf00843c58d52e788c08fd860, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/501d73e3b14e4cbb997e34912453243e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/da7cafb4f16445568ca2c8d2dcb04205, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/f2ed7c46c0fb448e82b8f6f45f835e54, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0c43bc3c89ff4f8596a97256c5ae70e0, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d1e0b472d454d708a38cf951912f7e6] to archive 2024-11-19T16:17:12,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:17:12,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/e7403480156f4915968799af262b7572 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/e7403480156f4915968799af262b7572 2024-11-19T16:17:12,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d0c4d61a0e64e35a4b34e841d7e0069 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d0c4d61a0e64e35a4b34e841d7e0069 2024-11-19T16:17:12,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/280ef379b0244d6787e204cfb030f41b to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/280ef379b0244d6787e204cfb030f41b 2024-11-19T16:17:12,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22217a72ff46446f91b5aac4564e0aa7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22217a72ff46446f91b5aac4564e0aa7 2024-11-19T16:17:12,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/2917e8de0d204f2481147cd77cda80e8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/2917e8de0d204f2481147cd77cda80e8 2024-11-19T16:17:12,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/efc55cda1be0490bb301d3393373b58e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/efc55cda1be0490bb301d3393373b58e 2024-11-19T16:17:12,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/a0ff963b636d43a8973e77df4edd59d8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/a0ff963b636d43a8973e77df4edd59d8 2024-11-19T16:17:12,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ba252afed0394c7e815c8ce8a88d376d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ba252afed0394c7e815c8ce8a88d376d 2024-11-19T16:17:12,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/68ebaa9a7b2d45db91844d697c8b6f8a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/68ebaa9a7b2d45db91844d697c8b6f8a 2024-11-19T16:17:12,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0a4b0d72187d4ed3b030cbb1540a7274 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0a4b0d72187d4ed3b030cbb1540a7274 2024-11-19T16:17:12,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/11567aeacaf242c7960cf8182023fc42 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/11567aeacaf242c7960cf8182023fc42 2024-11-19T16:17:12,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ea917fef73e847f2b26fd1a27ef0e9c1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ea917fef73e847f2b26fd1a27ef0e9c1 2024-11-19T16:17:12,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/96098c87254940fb8c53f09ad54d9cfe to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/96098c87254940fb8c53f09ad54d9cfe 2024-11-19T16:17:12,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/87317cbc64c04c16abf7114410d29039 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/87317cbc64c04c16abf7114410d29039 2024-11-19T16:17:12,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/715a2f4afbbb4021a67f8a1dec8a93af to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/715a2f4afbbb4021a67f8a1dec8a93af 2024-11-19T16:17:12,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ffb945437dea40849b142e149275cb96 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/ffb945437dea40849b142e149275cb96 2024-11-19T16:17:12,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22eed6384e5c4e419ee2e549dfc87bac to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/22eed6384e5c4e419ee2e549dfc87bac 2024-11-19T16:17:12,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/d7b84eb8b287488db151f75ff9ad3abc to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/d7b84eb8b287488db151f75ff9ad3abc 2024-11-19T16:17:12,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/efbf483ee06146acb04e84a6e6608bb6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/efbf483ee06146acb04e84a6e6608bb6 2024-11-19T16:17:12,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/07ee82faf00843c58d52e788c08fd860 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/07ee82faf00843c58d52e788c08fd860 2024-11-19T16:17:12,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/501d73e3b14e4cbb997e34912453243e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/501d73e3b14e4cbb997e34912453243e 2024-11-19T16:17:12,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/da7cafb4f16445568ca2c8d2dcb04205 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/da7cafb4f16445568ca2c8d2dcb04205 2024-11-19T16:17:12,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/f2ed7c46c0fb448e82b8f6f45f835e54 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/f2ed7c46c0fb448e82b8f6f45f835e54 2024-11-19T16:17:12,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0c43bc3c89ff4f8596a97256c5ae70e0 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0c43bc3c89ff4f8596a97256c5ae70e0 2024-11-19T16:17:12,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d1e0b472d454d708a38cf951912f7e6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/0d1e0b472d454d708a38cf951912f7e6 2024-11-19T16:17:12,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/e006825851024fa6857322b8246ae283, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f9e5baa0e00643fb98253d4ebe1efa9d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f2c06a63d0054ad5991fe82b81b40844, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8a7ac0b6dda644b8b0ff56a4d17a58bf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/34ec5189a9a441828bdfae309b34af9d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/42308408ca974f668accbb3d5ae396ba, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/6dbe7fa136254ea88795f4b37fb5fd0d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/fc0b389cf71b45e1bcda65a5f828eb19, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/bd8ea04c43ed4815a70a5ccb69b46a68, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/1de0f3d8033943b3a104348c330e1ebf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/312f21547fd148ccb2f2416d464d7856, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/da470c302b334971b489ffbccfda5ca8, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/e970d51f8da64a3e9a0276f53d025748, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5c619bec28cb4cf0aa15098f8f99fbcc, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/78c03585884646b8a076c93ebe6f088d, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/82189379da834333b255dc45a9e604b6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/23bfe436a74b4329af5920a3851770cb, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f964c3bee0f844138c7134f58be68d34, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8d02dbe582cd4579b611e91d6471c51a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/a3b42e0a3a2242068bf99724fb117e3a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/59e65446ccff462d818e8b96a891062e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/d1992a7f13d340499433905d5435b9f6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/25360e70f1a2421b80042053722372b3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5dc541f5ccc148be804082f9e3cf5aa4, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8771e30c7d814114bd8b56bb072fa41e] to archive 2024-11-19T16:17:12,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:17:12,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/e006825851024fa6857322b8246ae283 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/e006825851024fa6857322b8246ae283 2024-11-19T16:17:12,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f9e5baa0e00643fb98253d4ebe1efa9d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f9e5baa0e00643fb98253d4ebe1efa9d 2024-11-19T16:17:12,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f2c06a63d0054ad5991fe82b81b40844 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f2c06a63d0054ad5991fe82b81b40844 2024-11-19T16:17:12,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8a7ac0b6dda644b8b0ff56a4d17a58bf to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8a7ac0b6dda644b8b0ff56a4d17a58bf 2024-11-19T16:17:12,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/34ec5189a9a441828bdfae309b34af9d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/34ec5189a9a441828bdfae309b34af9d 2024-11-19T16:17:12,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/42308408ca974f668accbb3d5ae396ba to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/42308408ca974f668accbb3d5ae396ba 2024-11-19T16:17:12,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/6dbe7fa136254ea88795f4b37fb5fd0d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/6dbe7fa136254ea88795f4b37fb5fd0d 2024-11-19T16:17:12,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/fc0b389cf71b45e1bcda65a5f828eb19 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/fc0b389cf71b45e1bcda65a5f828eb19 2024-11-19T16:17:12,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/bd8ea04c43ed4815a70a5ccb69b46a68 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/bd8ea04c43ed4815a70a5ccb69b46a68 2024-11-19T16:17:12,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/1de0f3d8033943b3a104348c330e1ebf to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/1de0f3d8033943b3a104348c330e1ebf 2024-11-19T16:17:12,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/312f21547fd148ccb2f2416d464d7856 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/312f21547fd148ccb2f2416d464d7856 2024-11-19T16:17:12,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/da470c302b334971b489ffbccfda5ca8 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/da470c302b334971b489ffbccfda5ca8 2024-11-19T16:17:12,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/e970d51f8da64a3e9a0276f53d025748 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/e970d51f8da64a3e9a0276f53d025748 2024-11-19T16:17:12,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5c619bec28cb4cf0aa15098f8f99fbcc to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5c619bec28cb4cf0aa15098f8f99fbcc 2024-11-19T16:17:12,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/78c03585884646b8a076c93ebe6f088d to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/78c03585884646b8a076c93ebe6f088d 2024-11-19T16:17:12,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/82189379da834333b255dc45a9e604b6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/82189379da834333b255dc45a9e604b6 2024-11-19T16:17:12,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/23bfe436a74b4329af5920a3851770cb to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/23bfe436a74b4329af5920a3851770cb 2024-11-19T16:17:12,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f964c3bee0f844138c7134f58be68d34 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/f964c3bee0f844138c7134f58be68d34 2024-11-19T16:17:12,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8d02dbe582cd4579b611e91d6471c51a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8d02dbe582cd4579b611e91d6471c51a 2024-11-19T16:17:12,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/a3b42e0a3a2242068bf99724fb117e3a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/a3b42e0a3a2242068bf99724fb117e3a 2024-11-19T16:17:12,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/59e65446ccff462d818e8b96a891062e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/59e65446ccff462d818e8b96a891062e 2024-11-19T16:17:12,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/d1992a7f13d340499433905d5435b9f6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/d1992a7f13d340499433905d5435b9f6 2024-11-19T16:17:12,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/25360e70f1a2421b80042053722372b3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/25360e70f1a2421b80042053722372b3 2024-11-19T16:17:12,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5dc541f5ccc148be804082f9e3cf5aa4 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5dc541f5ccc148be804082f9e3cf5aa4 2024-11-19T16:17:12,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8771e30c7d814114bd8b56bb072fa41e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/8771e30c7d814114bd8b56bb072fa41e 2024-11-19T16:17:12,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a061afdbdf454a29a3239c824e21959a, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/df2e83dbef4c44ad9f32df6e5d6bba70, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/0706846e5eb541d5ae0a276525428446, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/39235d5096b849099199dfe533d1b8cf, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/2bb9a5111c63474aa50e15ff23d074e1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/1e8abaaa681f4773bec847b652a10202, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/04cee37b8b0e4c0fb0157901f524146c, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/40181433c80e45a1a32f0c6f6736eae5, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/234d87c8298440249376c0522fab7983, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/c55c8ecfd2fc44e39170f49c380ebd69, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/921cbfb66853424890bdea8ae72177f3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/7356de6343ba4eb791e5767da7730e19, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/39c03690f7a84f8da68081eaa99a23e3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/8d671518c60d47f4911bb28afebbf5c3, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/0d4a4e109d674ae5acd4c75a63808204, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a408e7591c4e459d8eefcd47b920ad34, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/ef1ef1cc0c1647e3bb041ce4ce2f80db, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/ddcb13a4f8e2456cb6b009ecc63da38f, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a904de6e0f2842bca9d148f2aca012d6, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/92d1d3d4251c4b01855232dcbbc06b53, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/039ef975ab194d16845decdb6d9f5150, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/e4f0159465344c739f9403a18f05f560, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/b72e9459642b481bbd2e4f385cec433e, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/31436a8f263d42048f848320499f62f1, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/cc0db331b2eb4e5e9dc3169115fc3470] to archive 2024-11-19T16:17:12,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T16:17:12,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a061afdbdf454a29a3239c824e21959a to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a061afdbdf454a29a3239c824e21959a 2024-11-19T16:17:12,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/df2e83dbef4c44ad9f32df6e5d6bba70 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/df2e83dbef4c44ad9f32df6e5d6bba70 2024-11-19T16:17:12,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/0706846e5eb541d5ae0a276525428446 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/0706846e5eb541d5ae0a276525428446 2024-11-19T16:17:12,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/39235d5096b849099199dfe533d1b8cf to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/39235d5096b849099199dfe533d1b8cf 2024-11-19T16:17:12,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/2bb9a5111c63474aa50e15ff23d074e1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/2bb9a5111c63474aa50e15ff23d074e1 2024-11-19T16:17:12,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/1e8abaaa681f4773bec847b652a10202 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/1e8abaaa681f4773bec847b652a10202 2024-11-19T16:17:12,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/04cee37b8b0e4c0fb0157901f524146c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/04cee37b8b0e4c0fb0157901f524146c 2024-11-19T16:17:12,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/40181433c80e45a1a32f0c6f6736eae5 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/40181433c80e45a1a32f0c6f6736eae5 2024-11-19T16:17:12,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/234d87c8298440249376c0522fab7983 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/234d87c8298440249376c0522fab7983 2024-11-19T16:17:12,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/c55c8ecfd2fc44e39170f49c380ebd69 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/c55c8ecfd2fc44e39170f49c380ebd69 2024-11-19T16:17:12,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/921cbfb66853424890bdea8ae72177f3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/921cbfb66853424890bdea8ae72177f3 2024-11-19T16:17:12,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/7356de6343ba4eb791e5767da7730e19 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/7356de6343ba4eb791e5767da7730e19 2024-11-19T16:17:12,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/39c03690f7a84f8da68081eaa99a23e3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/39c03690f7a84f8da68081eaa99a23e3 2024-11-19T16:17:12,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/8d671518c60d47f4911bb28afebbf5c3 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/8d671518c60d47f4911bb28afebbf5c3 2024-11-19T16:17:12,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/0d4a4e109d674ae5acd4c75a63808204 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/0d4a4e109d674ae5acd4c75a63808204 2024-11-19T16:17:12,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a408e7591c4e459d8eefcd47b920ad34 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a408e7591c4e459d8eefcd47b920ad34 2024-11-19T16:17:12,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/ef1ef1cc0c1647e3bb041ce4ce2f80db to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/ef1ef1cc0c1647e3bb041ce4ce2f80db 2024-11-19T16:17:12,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/ddcb13a4f8e2456cb6b009ecc63da38f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/ddcb13a4f8e2456cb6b009ecc63da38f 2024-11-19T16:17:12,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a904de6e0f2842bca9d148f2aca012d6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/a904de6e0f2842bca9d148f2aca012d6 2024-11-19T16:17:12,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/92d1d3d4251c4b01855232dcbbc06b53 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/92d1d3d4251c4b01855232dcbbc06b53 2024-11-19T16:17:12,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/039ef975ab194d16845decdb6d9f5150 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/039ef975ab194d16845decdb6d9f5150 2024-11-19T16:17:12,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/e4f0159465344c739f9403a18f05f560 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/e4f0159465344c739f9403a18f05f560 2024-11-19T16:17:12,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/b72e9459642b481bbd2e4f385cec433e to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/b72e9459642b481bbd2e4f385cec433e 2024-11-19T16:17:12,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/31436a8f263d42048f848320499f62f1 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/31436a8f263d42048f848320499f62f1 2024-11-19T16:17:12,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/cc0db331b2eb4e5e9dc3169115fc3470 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/cc0db331b2eb4e5e9dc3169115fc3470 2024-11-19T16:17:12,622 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/recovered.edits/450.seqid, newMaxSeqId=450, maxSeqId=4 2024-11-19T16:17:12,623 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. 2024-11-19T16:17:12,623 DEBUG [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 0dc20a97b5ebf3ba62589849ed17c46f: Waiting for close lock at 1732033030815Running coprocessor pre-close hooks at 1732033030815Disabling compacts and flushes for region at 1732033030815Disabling writes for close at 1732033030815Obtaining lock to block concurrent updates at 1732033030815Preparing flush snapshotting stores in 0dc20a97b5ebf3ba62589849ed17c46f at 1732033030815Finished memstore snapshotting TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732033030816 (+1 ms)Flushing stores of TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f. at 1732033030817 (+1 ms)Flushing 0dc20a97b5ebf3ba62589849ed17c46f/A: creating writer at 1732033030817Flushing mob file 0dc20a97b5ebf3ba62589849ed17c46f/A: appending metadata at 1732033030824 (+7 ms)Flushing mob file 0dc20a97b5ebf3ba62589849ed17c46f/A: closing flushed file at 1732033030824Flushing 0dc20a97b5ebf3ba62589849ed17c46f/A: appending metadata at 1732033031269 (+445 ms)Flushing 0dc20a97b5ebf3ba62589849ed17c46f/A: closing flushed file at 1732033031269Flushing 0dc20a97b5ebf3ba62589849ed17c46f/B: creating writer at 1732033031678 (+409 ms)Flushing 0dc20a97b5ebf3ba62589849ed17c46f/B: appending metadata at 1732033031681 (+3 ms)Flushing 0dc20a97b5ebf3ba62589849ed17c46f/B: closing flushed file at 1732033031681Flushing 0dc20a97b5ebf3ba62589849ed17c46f/C: creating writer at 1732033032094 (+413 ms)Flushing 0dc20a97b5ebf3ba62589849ed17c46f/C: appending metadata at 1732033032099 (+5 ms)Flushing 0dc20a97b5ebf3ba62589849ed17c46f/C: closing flushed file at 1732033032099Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52514160: reopening flushed file at 1732033032508 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79d1460d: reopening flushed file at 1732033032513 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fe3298e: reopening flushed file at 1732033032518 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0dc20a97b5ebf3ba62589849ed17c46f in 1707ms, sequenceid=447, compaction requested=true at 1732033032522 (+4 ms)Writing region close event to WAL at 1732033032617 (+95 ms)Running coprocessor post-close hooks at 1732033032622 (+5 ms)Closed at 1732033032622 2024-11-19T16:17:12,625 INFO [RS_CLOSE_REGION-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,626 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=0dc20a97b5ebf3ba62589849ed17c46f, regionState=CLOSED 2024-11-19T16:17:12,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 because future has completed 2024-11-19T16:17:12,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-19T16:17:12,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 0dc20a97b5ebf3ba62589849ed17c46f, server=32a8860bb9d6,43095,1732032854050 in 1.9670 sec 2024-11-19T16:17:12,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-19T16:17:12,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0dc20a97b5ebf3ba62589849ed17c46f, UNASSIGN in 1.9720 sec 2024-11-19T16:17:12,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-19T16:17:12,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9750 sec 2024-11-19T16:17:12,636 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732033032636"}]},"ts":"1732033032636"} 2024-11-19T16:17:12,637 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T16:17:12,637 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T16:17:12,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9860 sec 2024-11-19T16:17:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T16:17:12,818 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:17:12,819 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-19T16:17:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:17:12,823 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:17:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T16:17:12,825 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:17:12,828 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,833 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C, FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/recovered.edits] 2024-11-19T16:17:12,838 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/32f29aa88d184c5c854e1e7f8cb2c290 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/32f29aa88d184c5c854e1e7f8cb2c290 2024-11-19T16:17:12,839 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/a5a290649eb54f958f4b8644a97421ee to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/a5a290649eb54f958f4b8644a97421ee 2024-11-19T16:17:12,841 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/da6b022a90b94ec280598089998613a6 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/A/da6b022a90b94ec280598089998613a6 2024-11-19T16:17:12,843 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/491f95853aeb4b45b12cc1a605daf1a7 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/491f95853aeb4b45b12cc1a605daf1a7 2024-11-19T16:17:12,844 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5fc4ca6271414b8cbb0326fe77566416 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/5fc4ca6271414b8cbb0326fe77566416 2024-11-19T16:17:12,845 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/7dddb94d78e44beea2c19bb059334119 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/B/7dddb94d78e44beea2c19bb059334119 2024-11-19T16:17:12,847 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/55c172c2b4ca4c34b0053a67cb1d6193 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/55c172c2b4ca4c34b0053a67cb1d6193 2024-11-19T16:17:12,847 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/da97942a2ed1484eaa8c6ea7292e0c0c to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/da97942a2ed1484eaa8c6ea7292e0c0c 2024-11-19T16:17:12,848 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/faa40a12dec344fa940a717fa9f31f69 to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/C/faa40a12dec344fa940a717fa9f31f69 2024-11-19T16:17:12,850 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/recovered.edits/450.seqid to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f/recovered.edits/450.seqid 2024-11-19T16:17:12,851 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/default/TestAcidGuarantees/0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,851 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T16:17:12,851 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T16:17:12,852 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T16:17:12,854 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111910441db17e394799976a644b04d8c77c_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111910441db17e394799976a644b04d8c77c_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,855 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111911a23a8098ac4453926c18131aa5e214_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111911a23a8098ac4453926c18131aa5e214_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,856 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191bd183ef408d4136afa0cf19afdad502_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191bd183ef408d4136afa0cf19afdad502_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,858 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195d624e6464cf4ed594893955b5370d4a_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195d624e6464cf4ed594893955b5370d4a_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,859 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111965b21447b475488895935b865bae42be_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111965b21447b475488895935b865bae42be_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,860 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111978b51baef2db4ae3a532e0b4df2c00e7_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111978b51baef2db4ae3a532e0b4df2c00e7_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,861 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111981b0abe2e85e4729bc0054cb38a304f1_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111981b0abe2e85e4729bc0054cb38a304f1_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,862 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198b1bf15b2da14d9588cbb4602a3c5dae_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198b1bf15b2da14d9588cbb4602a3c5dae_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,863 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119904900ee1f0a48c3b7ccc6490e1946b9_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119904900ee1f0a48c3b7ccc6490e1946b9_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,864 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199454a12322c848a8943658a3e40f3636_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199454a12322c848a8943658a3e40f3636_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,865 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199da4f1c984274e95a3e22d831783624a_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199da4f1c984274e95a3e22d831783624a_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,866 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a052d91455024b2f9710703e1d5f394a_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a052d91455024b2f9710703e1d5f394a_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,867 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a2025a01d42540f59e18aed9db166588_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a2025a01d42540f59e18aed9db166588_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,868 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b0a52c3b05244a7b96d37ad0a5cebd7d_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b0a52c3b05244a7b96d37ad0a5cebd7d_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,869 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b22234e3d16f4a0ba9d14f0658160c63_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b22234e3d16f4a0ba9d14f0658160c63_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,870 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c3242a179d594b7a9f8e50b4b9535df8_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c3242a179d594b7a9f8e50b4b9535df8_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,870 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c79cc536ee7e40f085428b7472fd8da5_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c79cc536ee7e40f085428b7472fd8da5_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,871 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e7b8921a94b248f88fcfa670d810f447_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e7b8921a94b248f88fcfa670d810f447_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,872 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e9abdc8c99254ae68aa3bb1739de2d66_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e9abdc8c99254ae68aa3bb1739de2d66_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,873 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f02e507a9be84641a51a12feb2c06795_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f02e507a9be84641a51a12feb2c06795_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,874 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f9f25494d81d4171bb37940b1016b057_0dc20a97b5ebf3ba62589849ed17c46f to hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f9f25494d81d4171bb37940b1016b057_0dc20a97b5ebf3ba62589849ed17c46f 2024-11-19T16:17:12,874 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T16:17:12,876 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:17:12,878 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T16:17:12,880 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T16:17:12,881 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:17:12,881 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T16:17:12,881 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732033032881"}]},"ts":"9223372036854775807"} 2024-11-19T16:17:12,882 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T16:17:12,882 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0dc20a97b5ebf3ba62589849ed17c46f, NAME => 'TestAcidGuarantees,,1732033002694.0dc20a97b5ebf3ba62589849ed17c46f.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T16:17:12,882 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T16:17:12,882 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732033032882"}]},"ts":"9223372036854775807"} 2024-11-19T16:17:12,883 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T16:17:12,884 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T16:17:12,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-11-19T16:17:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T16:17:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T16:17:12,933 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T16:17:12,940 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 250), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=450 (was 453), ProcessCount=11 (was 11), AvailableMemoryMB=4594 (was 4062) - AvailableMemoryMB LEAK? - 2024-11-19T16:17:12,940 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T16:17:12,941 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T16:17:12,941 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T16:17:12,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:12,941 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T16:17:12,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:12,942 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T16:17:12,942 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2117780854, stopped=false 2024-11-19T16:17:12,942 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=32a8860bb9d6,37355,1732032853237 2024-11-19T16:17:12,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T16:17:12,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T16:17:12,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:17:12,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:17:12,952 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T16:17:12,952 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T16:17:12,953 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T16:17:12,954 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T16:17:12,954 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T16:17:12,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:12,955 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32a8860bb9d6,43095,1732032854050' ***** 2024-11-19T16:17:12,955 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T16:17:12,955 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T16:17:12,955 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T16:17:12,955 INFO [RS:0;32a8860bb9d6:43095 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T16:17:12,955 INFO [RS:0;32a8860bb9d6:43095 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T16:17:12,955 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(959): stopping server 32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:12,956 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T16:17:12,956 INFO [RS:0;32a8860bb9d6:43095 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;32a8860bb9d6:43095. 2024-11-19T16:17:12,956 DEBUG [RS:0;32a8860bb9d6:43095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T16:17:12,956 DEBUG [RS:0;32a8860bb9d6:43095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:12,956 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T16:17:12,956 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T16:17:12,956 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T16:17:12,956 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T16:17:12,957 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T16:17:12,957 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T16:17:12,957 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T16:17:12,957 INFO [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T16:17:12,957 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T16:17:12,957 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T16:17:12,957 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T16:17:12,957 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T16:17:12,957 INFO [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-19T16:17:12,978 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/.tmp/info/b511be4afeed495d9c675eae67388e0a is 95, key is TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88./info:/1732032884929/DeleteFamily/seqid=0 2024-11-19T16:17:12,980 INFO [regionserver/32a8860bb9d6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T16:17:12,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742482_1658 (size=6514) 2024-11-19T16:17:13,157 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T16:17:13,358 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T16:17:13,382 INFO [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/.tmp/info/b511be4afeed495d9c675eae67388e0a 2024-11-19T16:17:13,399 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/.tmp/ns/2f6f38b0e456410a8d81bb819e567695 is 93, key is TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88./ns:/1732032884888/DeleteFamily/seqid=0 2024-11-19T16:17:13,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742483_1659 (size=5906) 2024-11-19T16:17:13,404 INFO [regionserver/32a8860bb9d6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T16:17:13,404 INFO [regionserver/32a8860bb9d6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T16:17:13,558 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T16:17:13,598 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T16:17:13,758 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T16:17:13,803 INFO [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/.tmp/ns/2f6f38b0e456410a8d81bb819e567695 2024-11-19T16:17:13,823 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/.tmp/rep_barrier/32fed3f04ff34402af0e48b9fcc58707 is 102, key is TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88./rep_barrier:/1732032884888/DeleteFamily/seqid=0 2024-11-19T16:17:13,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742484_1660 (size=6025) 2024-11-19T16:17:13,959 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T16:17:14,159 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T16:17:14,226 INFO [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/.tmp/rep_barrier/32fed3f04ff34402af0e48b9fcc58707 2024-11-19T16:17:14,256 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/.tmp/table/4af5c13ff33c491f94d365eda05699cd is 96, key is TestAcidGuarantees,,1732032859306.c20cca7a0cf0913bda0c7ed239b24e88./table:/1732032884888/DeleteFamily/seqid=0 2024-11-19T16:17:14,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742485_1661 (size=5936) 2024-11-19T16:17:14,359 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T16:17:14,560 DEBUG [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T16:17:14,660 INFO [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/.tmp/table/4af5c13ff33c491f94d365eda05699cd 2024-11-19T16:17:14,671 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/.tmp/info/b511be4afeed495d9c675eae67388e0a as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/info/b511be4afeed495d9c675eae67388e0a 2024-11-19T16:17:14,676 INFO [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/info/b511be4afeed495d9c675eae67388e0a, entries=12, sequenceid=90, filesize=6.4 K 2024-11-19T16:17:14,676 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/.tmp/ns/2f6f38b0e456410a8d81bb819e567695 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/ns/2f6f38b0e456410a8d81bb819e567695 2024-11-19T16:17:14,680 INFO [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/ns/2f6f38b0e456410a8d81bb819e567695, entries=8, sequenceid=90, filesize=5.8 K 2024-11-19T16:17:14,681 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/.tmp/rep_barrier/32fed3f04ff34402af0e48b9fcc58707 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/rep_barrier/32fed3f04ff34402af0e48b9fcc58707 2024-11-19T16:17:14,684 INFO [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/rep_barrier/32fed3f04ff34402af0e48b9fcc58707, entries=6, sequenceid=90, filesize=5.9 K 2024-11-19T16:17:14,685 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/.tmp/table/4af5c13ff33c491f94d365eda05699cd as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/table/4af5c13ff33c491f94d365eda05699cd 2024-11-19T16:17:14,688 INFO [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/table/4af5c13ff33c491f94d365eda05699cd, entries=7, sequenceid=90, filesize=5.8 K 2024-11-19T16:17:14,689 INFO [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1732ms, sequenceid=90, compaction requested=false 2024-11-19T16:17:14,693 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-19T16:17:14,694 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T16:17:14,694 INFO [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T16:17:14,694 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732033032957Running coprocessor pre-close hooks at 1732033032957Disabling compacts and flushes for region at 1732033032957Disabling writes for close at 1732033032957Obtaining lock to block concurrent updates at 1732033032957Preparing flush snapshotting stores in 1588230740 at 1732033032957Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732033032958 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732033032959 (+1 ms)Flushing 1588230740/info: creating writer at 1732033032959Flushing 1588230740/info: appending metadata at 1732033032978 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732033032978Flushing 1588230740/ns: creating writer at 1732033033385 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732033033399 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732033033399Flushing 1588230740/rep_barrier: creating writer at 1732033033808 (+409 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732033033822 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732033033822Flushing 1588230740/table: creating writer at 1732033034233 (+411 ms)Flushing 1588230740/table: appending metadata at 1732033034255 (+22 ms)Flushing 1588230740/table: closing flushed file at 1732033034255Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@750626d3: reopening flushed file at 1732033034669 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37c63a4e: reopening flushed file at 1732033034676 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58c6d377: reopening flushed file at 1732033034680 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ef1d49: reopening flushed file at 1732033034684 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1732ms, sequenceid=90, compaction requested=false at 1732033034689 (+5 ms)Writing region close event to WAL at 1732033034691 (+2 ms)Running coprocessor post-close hooks at 1732033034694 (+3 ms)Closed at 1732033034694 2024-11-19T16:17:14,694 DEBUG [RS_CLOSE_META-regionserver/32a8860bb9d6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T16:17:14,760 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(976): stopping server 32a8860bb9d6,43095,1732032854050; all regions closed. 2024-11-19T16:17:14,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741834_1010 (size=25344) 2024-11-19T16:17:14,766 DEBUG [RS:0;32a8860bb9d6:43095 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/oldWALs 2024-11-19T16:17:14,766 INFO [RS:0;32a8860bb9d6:43095 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32a8860bb9d6%2C43095%2C1732032854050.meta:.meta(num 1732032858608) 2024-11-19T16:17:14,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741832_1008 (size=18767782) 2024-11-19T16:17:14,769 DEBUG [RS:0;32a8860bb9d6:43095 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/oldWALs 2024-11-19T16:17:14,770 INFO [RS:0;32a8860bb9d6:43095 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32a8860bb9d6%2C43095%2C1732032854050:(num 1732032857562) 2024-11-19T16:17:14,770 DEBUG [RS:0;32a8860bb9d6:43095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T16:17:14,770 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T16:17:14,770 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T16:17:14,770 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.ChoreService(370): Chore service for: regionserver/32a8860bb9d6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T16:17:14,770 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T16:17:14,770 INFO [regionserver/32a8860bb9d6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T16:17:14,777 INFO [RS:0;32a8860bb9d6:43095 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43095 2024-11-19T16:17:14,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32a8860bb9d6,43095,1732032854050 2024-11-19T16:17:14,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T16:17:14,832 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T16:17:14,842 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32a8860bb9d6,43095,1732032854050] 2024-11-19T16:17:14,852 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32a8860bb9d6,43095,1732032854050 already deleted, retry=false 2024-11-19T16:17:14,853 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 32a8860bb9d6,43095,1732032854050 expired; onlineServers=0 2024-11-19T16:17:14,853 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '32a8860bb9d6,37355,1732032853237' ***** 2024-11-19T16:17:14,853 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T16:17:14,853 INFO [M:0;32a8860bb9d6:37355 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T16:17:14,853 INFO [M:0;32a8860bb9d6:37355 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T16:17:14,853 DEBUG [M:0;32a8860bb9d6:37355 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T16:17:14,853 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T16:17:14,853 DEBUG [M:0;32a8860bb9d6:37355 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T16:17:14,853 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster-HFileCleaner.small.0-1732032857293 {}] cleaner.HFileCleaner(306): Exit Thread[master/32a8860bb9d6:0:becomeActiveMaster-HFileCleaner.small.0-1732032857293,5,FailOnTimeoutGroup] 2024-11-19T16:17:14,853 DEBUG [master/32a8860bb9d6:0:becomeActiveMaster-HFileCleaner.large.0-1732032857283 {}] cleaner.HFileCleaner(306): Exit Thread[master/32a8860bb9d6:0:becomeActiveMaster-HFileCleaner.large.0-1732032857283,5,FailOnTimeoutGroup] 2024-11-19T16:17:14,853 INFO [M:0;32a8860bb9d6:37355 {}] hbase.ChoreService(370): Chore service for: master/32a8860bb9d6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T16:17:14,853 INFO [M:0;32a8860bb9d6:37355 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T16:17:14,854 DEBUG [M:0;32a8860bb9d6:37355 {}] master.HMaster(1795): Stopping service threads 2024-11-19T16:17:14,854 INFO [M:0;32a8860bb9d6:37355 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T16:17:14,854 INFO [M:0;32a8860bb9d6:37355 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T16:17:14,854 ERROR [M:0;32a8860bb9d6:37355 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-19T16:17:14,854 INFO [M:0;32a8860bb9d6:37355 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T16:17:14,854 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T16:17:14,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T16:17:14,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T16:17:14,863 DEBUG [M:0;32a8860bb9d6:37355 {}] zookeeper.ZKUtil(347): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T16:17:14,863 WARN [M:0;32a8860bb9d6:37355 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T16:17:14,864 INFO [M:0;32a8860bb9d6:37355 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/.lastflushedseqids 2024-11-19T16:17:14,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742486_1662 (size=72) 2024-11-19T16:17:14,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T16:17:14,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43095-0x10153dfed330001, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T16:17:14,943 INFO [RS:0;32a8860bb9d6:43095 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T16:17:14,943 INFO [RS:0;32a8860bb9d6:43095 {}] regionserver.HRegionServer(1031): Exiting; stopping=32a8860bb9d6,43095,1732032854050; zookeeper connection closed. 2024-11-19T16:17:14,943 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2067e16f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2067e16f 2024-11-19T16:17:14,944 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T16:17:15,273 INFO [M:0;32a8860bb9d6:37355 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T16:17:15,274 INFO [M:0;32a8860bb9d6:37355 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T16:17:15,274 DEBUG [M:0;32a8860bb9d6:37355 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T16:17:15,274 INFO [M:0;32a8860bb9d6:37355 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T16:17:15,274 DEBUG [M:0;32a8860bb9d6:37355 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T16:17:15,274 DEBUG [M:0;32a8860bb9d6:37355 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T16:17:15,274 DEBUG [M:0;32a8860bb9d6:37355 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T16:17:15,275 INFO [M:0;32a8860bb9d6:37355 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=766.14 KB heapSize=937.35 KB 2024-11-19T16:17:15,296 DEBUG [M:0;32a8860bb9d6:37355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cd31654d20e04ff58ad0b77d045d3ee7 is 82, key is hbase:meta,,1/info:regioninfo/1732032858689/Put/seqid=0 2024-11-19T16:17:15,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742487_1663 (size=5672) 2024-11-19T16:17:15,700 INFO [M:0;32a8860bb9d6:37355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2112 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cd31654d20e04ff58ad0b77d045d3ee7 2024-11-19T16:17:15,726 DEBUG [M:0;32a8860bb9d6:37355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/59eec97e20d84e45b77665fa61d42fea is 2480, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1732032887739/Put/seqid=0 2024-11-19T16:17:15,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742488_1664 (size=45081) 2024-11-19T16:17:16,131 INFO [M:0;32a8860bb9d6:37355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=765.58 KB at sequenceid=2112 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/59eec97e20d84e45b77665fa61d42fea 2024-11-19T16:17:16,135 INFO [M:0;32a8860bb9d6:37355 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 59eec97e20d84e45b77665fa61d42fea 2024-11-19T16:17:16,154 DEBUG [M:0;32a8860bb9d6:37355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10a9d8012ac74b1692ade966dc21f8c1 is 69, key is 32a8860bb9d6,43095,1732032854050/rs:state/1732032857321/Put/seqid=0 2024-11-19T16:17:16,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073742489_1665 (size=5156) 2024-11-19T16:17:16,557 INFO [M:0;32a8860bb9d6:37355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2112 (bloomFilter=true), to=hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10a9d8012ac74b1692ade966dc21f8c1 2024-11-19T16:17:16,564 DEBUG [M:0;32a8860bb9d6:37355 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cd31654d20e04ff58ad0b77d045d3ee7 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cd31654d20e04ff58ad0b77d045d3ee7 2024-11-19T16:17:16,569 INFO [M:0;32a8860bb9d6:37355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cd31654d20e04ff58ad0b77d045d3ee7, entries=8, sequenceid=2112, filesize=5.5 K 2024-11-19T16:17:16,569 DEBUG [M:0;32a8860bb9d6:37355 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/59eec97e20d84e45b77665fa61d42fea as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/59eec97e20d84e45b77665fa61d42fea 2024-11-19T16:17:16,572 INFO [M:0;32a8860bb9d6:37355 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 59eec97e20d84e45b77665fa61d42fea 2024-11-19T16:17:16,572 INFO [M:0;32a8860bb9d6:37355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/59eec97e20d84e45b77665fa61d42fea, entries=168, sequenceid=2112, filesize=44.0 K 2024-11-19T16:17:16,573 DEBUG [M:0;32a8860bb9d6:37355 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10a9d8012ac74b1692ade966dc21f8c1 as hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/10a9d8012ac74b1692ade966dc21f8c1 2024-11-19T16:17:16,576 INFO [M:0;32a8860bb9d6:37355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36477/user/jenkins/test-data/2052a68f-9fd9-3f86-5be8-525d4ba5ce70/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/10a9d8012ac74b1692ade966dc21f8c1, entries=1, sequenceid=2112, filesize=5.0 K 2024-11-19T16:17:16,577 INFO [M:0;32a8860bb9d6:37355 {}] regionserver.HRegion(3140): Finished flush of dataSize ~766.14 KB/784524, heapSize ~937.05 KB/959544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1303ms, sequenceid=2112, compaction requested=false 2024-11-19T16:17:16,578 INFO [M:0;32a8860bb9d6:37355 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T16:17:16,579 DEBUG [M:0;32a8860bb9d6:37355 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732033035274Disabling compacts and flushes for region at 1732033035274Disabling writes for close at 1732033035274Obtaining lock to block concurrent updates at 1732033035275 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732033035275Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=784524, getHeapSize=959784, getOffHeapSize=0, getCellsCount=2305 at 1732033035275Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732033035277 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732033035277Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732033035296 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732033035296Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732033035705 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732033035725 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732033035725Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732033036135 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732033036153 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732033036153Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b2e0f1: reopening flushed file at 1732033036563 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11af2ed2: reopening flushed file at 1732033036569 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e200334: reopening flushed file at 1732033036573 (+4 ms)Finished flush of dataSize ~766.14 KB/784524, heapSize ~937.05 KB/959544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1303ms, sequenceid=2112, compaction requested=false at 1732033036577 (+4 ms)Writing region close event to WAL at 1732033036578 (+1 ms)Closed at 1732033036578 2024-11-19T16:17:16,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41553 is added to blk_1073741830_1006 (size=922995) 2024-11-19T16:17:16,582 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T16:17:16,582 INFO [M:0;32a8860bb9d6:37355 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T16:17:16,582 INFO [M:0;32a8860bb9d6:37355 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37355 2024-11-19T16:17:16,582 INFO [M:0;32a8860bb9d6:37355 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T16:17:16,732 INFO [M:0;32a8860bb9d6:37355 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T16:17:16,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T16:17:16,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10153dfed330000, quorum=127.0.0.1:50373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T16:17:16,738 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T16:17:16,742 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T16:17:16,742 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T16:17:16,742 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T16:17:16,743 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/hadoop.log.dir/,STOPPED} 2024-11-19T16:17:16,747 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T16:17:16,747 WARN [BP-1730201524-172.17.0.3-1732032849382 heartbeating to localhost/127.0.0.1:36477 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T16:17:16,747 WARN [BP-1730201524-172.17.0.3-1732032849382 heartbeating to localhost/127.0.0.1:36477 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1730201524-172.17.0.3-1732032849382 (Datanode Uuid f686d3cb-c1c2-4c17-b87a-b78090b56ff8) service to localhost/127.0.0.1:36477 2024-11-19T16:17:16,747 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T16:17:16,750 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/cluster_f5154a92-f5a6-8acf-0581-31f31135a66d/data/data1/current/BP-1730201524-172.17.0.3-1732032849382 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T16:17:16,750 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/cluster_f5154a92-f5a6-8acf-0581-31f31135a66d/data/data2/current/BP-1730201524-172.17.0.3-1732032849382 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T16:17:16,751 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T16:17:16,757 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T16:17:16,758 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T16:17:16,758 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T16:17:16,758 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T16:17:16,758 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8ddf7a43-3211-6c43-c172-90fd25a35918/hadoop.log.dir/,STOPPED} 2024-11-19T16:17:16,773 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T16:17:16,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down